scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-14 04:09:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 97

Merge remote-tracking branch 'origin/main' into litellm_bedrock_output_file_listing

Browse source
This commit is contained in:
Devin AI 2026-10-08 21:00:26 +00:00
commit 1b605df2d3
638 changed files with 51375 additions and 21282 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

140
.circleci/config.yml
View file

@ -1095,50 +1095,6 @@ jobs:
- google_generate_content_endpoint_coverage.xml
- google_generate_content_endpoint_coverage
llm_responses_api_testing:
docker:
- *python312_image
working_directory: ~/project
resource_class: large
environment:
REQUEST_TIMEOUT: "180"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- restore_cache:
keys:
- v1-uv-cache-{{ checksum "uv.lock" }}
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- save_cache:
paths:
- ~/.cache/uv
key: v1-uv-cache-{{ checksum "uv.lock" }}
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-v \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 8 \
--reruns 1 --only-rerun Timeout"
no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
ocr_testing:
docker:
- *python312_image
@ -2074,98 +2030,6 @@ jobs:
# Store test results
- store_test_results:
path: test-results
proxy_spend_accuracy_tests:
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- start_postgres
- start_redis
- start_fake_openai_endpoint
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# Point the proxy at the job-local Redis (start_redis) instead of the
# shared remote Redis. The Redis transaction buffer uses a single
# global pod-lock key (cronjob_lock:db_spend_update_job) and a single
# global buffer list (litellm_spend_update_buffer); sharing those
# across concurrent CI pipelines causes spend flushes to stall or
# land in the wrong DB, which is what makes this test flaky.
command: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=host.docker.internal \
-e REDIS_PORT=6379 \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
-e PROXY_BATCH_WRITE_AT=2 \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
background: true
- wait_for_service:
url: http://localhost:4000
timeout: "300"
- run:
name: Run tests
command: |
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-vv \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- store_test_results:
path: test-results
- run:
name: Stop and remove first container
when: always
command: |
docker stop my-app
docker rm my-app
docker stop redis-cache
docker rm redis-cache
proxy_multi_instance_tests:
machine:
image: ubuntu-2204:2024.04.1
@ -3591,9 +3455,6 @@ workflows:
- proxy_logging_guardrails_model_info_tests:
requires:
- build_docker_database_image
- proxy_spend_accuracy_tests:
requires:
- build_docker_database_image
- proxy_multi_instance_tests:
requires:
- build_docker_database_image
@ -3611,7 +3472,6 @@ workflows:
- realtime_translation_testing
- guardrails_testing
- google_generate_content_endpoint_testing
- llm_responses_api_testing
- ocr_testing
- search_testing
- batches_testing

4
.github/workflows/_test-unit-base.yml vendored
View file

@ -143,9 +143,9 @@ jobs:
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"

6
.github/workflows/helm_unit_test.yml vendored
View file

@ -22,6 +22,12 @@ jobs:
with:
persist-credentials: false
- name: Check Lens Compose configuration
run: |
python3 -m unittest discover -s deploy/lens -p 'test_*.py'
python3 deploy/lens/configure.py --version 1.2.3 --env-file "$RUNNER_TEMP/lens.env"
docker compose --env-file "$RUNNER_TEMP/lens.env" -f deploy/lens/stack.yaml config --quiet
- name: Set up Helm 3.11.1
uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
with:

65
.github/workflows/publish-basedpyright-base-counts.yml vendored
View file

@ -1,65 +0,0 @@
name: Publish basedpyright base counts
# Every commit on main can become a future merge-base.
# Publishing its per-rule basedpyright counts as an artifact lets
# scripts/type_check_gate.py download them in seconds instead of paying a
# 60-110s second basedpyright pass on every fresh worktree or moved merge-base.
# No concurrency group on purpose: runs must never cancel each other, because
# every sha's artifact matters (any of them can become a merge-base).
on:
push:
branches:
- main
workflow_dispatch:
inputs:
ref:
description: "Ref to compute and publish base counts for (defaults to the workflow run's commit)"
required: false
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ inputs.ref || github.sha }}
clean: true
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
# The gate provisions its own measurement env (.venv-typecheck: a frozen
# uv sync of its canonical dependency groups plus a generated Prisma
# client), so no install step here can drift from what local runs measure.
- name: Emit basedpyright counts for HEAD
run: |
python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts"
counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json)
echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
- name: Upload counts artifact
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: ${{ env.COUNTS_ARTIFACT_NAME }}
path: ${{ runner.temp }}/basedpyright-counts/
if-no-files-found: error

96
.github/workflows/publish-lint-base-counts.yml vendored Normal file
View file

@ -0,0 +1,96 @@
name: Publish lint base counts
# Every commit on main can become a future merge-base.
# Publishing its per-rule counts for each lint gate (strict ruff, type discipline,
# test quality, basedpyright) as an artifact lets the gates download them through
# scripts/lint_base_counts.py in seconds instead of scanning the merge-base tree in
# a throwaway worktree on every fresh checkout or moved merge-base.
# No concurrency group on purpose: runs must never cancel each other, because
# every sha's artifact matters (any of them can become a merge-base).
on:
push:
branches:
- main
workflow_dispatch:
inputs:
ref:
description: "Ref to compute and publish base counts for (defaults to the workflow run's commit)"
required: false
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
include:
- checker: ruff-strict
script: scripts/ruff_strict_gate.py
- checker: type-discipline
script: scripts/type_discipline_gate.py
- checker: test-quality
script: scripts/test_quality_gate.py
- checker: basedpyright
script: scripts/type_check_gate.py
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ inputs.ref || github.sha }}
clean: true
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
if: matrix.checker == 'basedpyright'
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
if: matrix.checker == 'basedpyright'
uses: ./.github/actions/cache-prisma-binaries
# The three source scanners only need the pinned dev tools (ruff and the
# stdlib checkers), the same versions test-linting.yml's lint job runs.
- name: Install the dev tools
if: matrix.checker != 'basedpyright'
run: |
uv sync --frozen --only-group dev --no-install-project
- name: Emit ${{ matrix.checker }} counts for HEAD
if: matrix.checker != 'basedpyright'
run: |
uv run --no-sync python ${{ matrix.script }} --emit-counts-dir "$RUNNER_TEMP/lint-counts"
# The basedpyright gate provisions its own measurement env (.venv-typecheck:
# a frozen uv sync of its canonical dependency groups plus a generated Prisma
# client), so no install step here can drift from what local runs measure.
- name: Emit basedpyright counts for HEAD
if: matrix.checker == 'basedpyright'
run: |
python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/lint-counts"
- name: Name the counts artifact
run: |
counts_file=$(ls "$RUNNER_TEMP"/lint-counts/${{ matrix.checker }}-counts-*.json)
echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
- name: Upload counts artifact
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: ${{ env.COUNTS_ARTIFACT_NAME }}
path: ${{ runner.temp }}/lint-counts/
if-no-files-found: error

54
.github/workflows/test-linting.yml vendored
View file

@ -17,9 +17,9 @@ jobs:
lint:
runs-on: ubuntu-latest
timeout-minutes: 15
# actions: read lets scripts/type_check_gate.py download the base-counts
# artifact published by publish-basedpyright-base-counts.yml instead of
# re-running basedpyright over the merge-base tree.
# actions: read lets the four lint gates download the base-counts artifacts
# published by publish-lint-base-counts.yml instead of re-scanning the
# merge-base tree in a throwaway worktree.
permissions:
contents: read
pull-requests: read
@ -144,18 +144,24 @@ jobs:
run: |
uv run --no-sync ruff check --config ruff-tests.toml tests
- name: Check strict-rule budget (delta vs base)
- name: Check strict ruff rules (delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
- name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
- name: Check type discipline (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
- name: Check test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs base)
- name: Check test quality (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA"
@ -164,7 +170,7 @@ jobs:
run: |
uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Check basedpyright budget (delta vs base)
- name: Check basedpyright (delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
@ -206,40 +212,6 @@ jobs:
run: |
uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Intentionally NON-GATING. This job turns red when a *-budget.json ceiling is
# raised (or a rule/budget is dropped) so a loosening is obvious in review, but it
# must be kept OUT of the branch-protection required-checks list so a justified
# bump can still be merged by a human who has seen and accepted the red.
budget-ratchet:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 1
persist-credentials: false
- name: Fetch ratchet base
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
retry git fetch --no-tags --depth=1 origin "$BASE_SHA"
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Ratchet check (budgets may only decrease; non-gating)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
python scripts/budget_ratchet_check.py --base "$BASE_SHA"
secret-scan:
runs-on: ubuntu-latest
timeout-minutes: 5

3
.github/workflows/test-rust.yml vendored
View file

@ -98,6 +98,9 @@ jobs:
rust-test:
runs-on: ubuntu-latest
timeout-minutes: 30
# Trim debuginfo to keep the job's target dir within the runner's disk
env:
CARGO_PROFILE_DEV_DEBUG: line-tables-only
defaults:
run:
working-directory: litellm-rust

10
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -141,6 +141,16 @@ jobs:
dist: loadscope
timeout: 15
- test-group: mcp-oauth
test-path: >-
tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
tests/unit/proxy/_experimental/mcp_server/outbound_credentials
workers: 4
dist: loadscope
timeout: 15
# ---- logging: split into 2 shards ----
- test-group: custom-logging
test-path: >-

10
.github/workflows/test-unit.yml vendored
View file

@ -346,6 +346,16 @@ jobs:
timeout-minutes: 60
job-timeout-minutes: 100
- shard: mcp-elicitation
artifact-name: mcp-elicitation
test-path: >-
tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
workers: 2
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-infra
artifact-name: proxy-infra
test-path: >-

9
AGENTS.md
View file

@ -54,13 +54,13 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
Python max line length is 120, not 88
Never edit or commit `ruff-strict-budget.json`, `type-discipline-budget.json`, `basedpyright-code-budget.json`, or `test-quality-budget.json` on a PR branch, and don't run `make lint-budget-update` there. A scheduled Devin automation lowers the limits on the default branch in its own PR by exactly what landed since the last ratchet, so concurrent PRs don't fight over the same `"limit"` lines. Keep the hosted automation's target in sync when the repository default changes. If your branch already carries a budget edit, drop it before opening the PR
The four lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/test_quality_gate.py`, `scripts/type_check_gate.py`) compare each rule's codebase count on your branch against the count at its merge-base with the default branch, and a rule may not grow. There are no budget files to edit or ratchet: when a gate fails, fix the violations the branch introduced or remove at least as many of that rule elsewhere in the tree. The one exception is `reportAny` / `reportExplicitAny`, which share a fixed codebase-wide cap in `ANY_CAPS` in `scripts/type_check_gate.py` because Any spreads past the lines you touch. A branch may add Anys while the total stays under that cap. Only lower `ANY_CAPS`, and only in its own PR to the default branch, never raise it on a feature branch
`make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone budget gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their cap in `ANY_CAPS`, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
If you get an LIT001 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
@ -97,6 +97,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
- Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
- Qualify every TypedDict field with `ReadOnly[...]` (LIT012), which nests freely with `Required` / `NotRequired` / `Annotated` in any order. If making the key writable is truly unavoidable, suppress with `# writable-ok: <reason>`
- Comprehensions take at most one `for` clause and one `if` clause (LIT014); split stacked clauses into a helper generator, a named intermediate, or a plain loop. Suppress with `# comprehension-ok: <reason>` only when unavoidable
- Every pydantic model must be frozen (LIT015), set via `model_config = ConfigDict(frozen=True)`, a dict-literal `model_config`, an inner `class Config`, or the class keywords. Subclasses inherit it unless they override it. Replace in-place field writes with `model_copy(update=...)`. If making the model mutable is truly unavoidable, suppress with `# frozen-ok: <reason>` on the `class` line
- Use dependency injection
- Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
- Use tagged unions + match
@ -129,4 +130,4 @@ Before implementing:
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify
Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or lint budgets to satisfy a check
Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or raise `ANY_CAPS` to satisfy a check

39
Makefile
View file

@ -6,9 +6,8 @@
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
test-rust-extension rust-sqlx-prepare lens-dev \
info lint lint-inner lint-dev lint-checks format \
lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
lint-test-quality lint-test-quality-budget-update \
lint-basedpyright lint-e2e-basedpyright lint-type-discipline \
lint-ruff-strict lint-gate lint-test-quality \
install-dev install-proxy-dev install-test-deps install-hooks \
install-helm-unittest check-circular-imports check-import-safety check check-inner pre-commit \
lint-install lint-fetch-base bootstrap
@ -32,13 +31,10 @@ help:
@echo " make lint-ruff - Run Ruff linting only"
@echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts"
@echo " make lint-e2e-basedpyright - Run basedpyright over tests/e2e and tests/e2e_harness (zero errors allowed)"
@echo " make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed"
@echo " make lint-format - Check ruff format formatting (matches CI)"
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
@echo " make lint-ruff-strict - Gate each strict ruff rule's codebase total against its merge-base count"
@echo " make lint-gate - Strict ruff gate in CI-parity mode (fetches the default branch, simulates the merge)"
@echo " make lint-ruff-budget-update - Ratchet ruff-strict-budget.json limits down by what this branch fixed"
@echo " make lint-test-quality - Gate the test suite against test-quality-budget.json"
@echo " make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + test quality + basedpyright)"
@echo " make lint-test-quality - Gate the test suite's TQ counts against their merge-base counts"
@echo " make check-circular-imports - Check for circular imports"
@echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
@ -143,7 +139,7 @@ lint-fetch-base:
# Mirror test-linting.yml's lint job environment: the proxy-dev group plus a generated
# Prisma client, so `basedpyright tests/e2e` resolves the same modules CI does. The
# budget gate itself no longer measures here (scripts/type_check_gate.py provisions its
# basedpyright gate itself no longer measures here (scripts/type_check_gate.py provisions its
# own .venv-typecheck). --inexact tops up the venv instead of pruning the proxy extras
# gen:api and the running proxy need.
lint-install:
@ -213,25 +209,20 @@ lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
$(UV_RUN) basedpyright tests/e2e tests/e2e_harness
# Type-discipline budget (mutable collections / casts / type guards / kwargs /
# Type-discipline gate (mutable collections / casts / type guards / kwargs /
# unexplained suppressions), the test-linting.yml step `make lint` used to omit.
lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/type_discipline_gate.py --base "$(BASE_REF)"
# Test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
# Test-quality gate (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
# litellm module-global mutation, credential-gated skips, conftest snapshot
# inventory), counted across tests/ the same delta-vs-base way.
lint-test-quality: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/test_quality_gate.py --base "$(BASE_REF)"
# --update lowers each limit by what this branch fixed since its branch point, so
# it needs the base ref fetched to resolve the merge-base.
lint-basedpyright-budget-update: install-dev
$(UV_RUN) python scripts/type_check_gate.py --update --base "$(BASE_REF)"
lint-format: format-check
lint-ruff-budget: install-dev
lint-ruff-strict: install-dev
$(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)"
# Strict gate, invoked the same way CI does in test-linting.yml so a local pass
@ -239,18 +230,6 @@ lint-ruff-budget: install-dev
lint-gate: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)"
lint-ruff-budget-update: install-dev
$(UV_RUN) python scripts/ruff_strict_gate.py --update --base "$(BASE_REF)"
lint-type-discipline-budget-update: install-dev
$(UV_RUN) python scripts/type_discipline_gate.py --update --base "$(BASE_REF)"
lint-test-quality-budget-update: install-dev
$(UV_RUN) python scripts/test_quality_gate.py --update --base "$(BASE_REF)"
# Ratchet all budgets in one shot (ruff strict + type-discipline + test quality + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-test-quality-budget-update lint-basedpyright-budget-update
check-circular-imports: $(LINT_DEP_INSTALL)
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
@ -260,7 +239,7 @@ check-import-safety: $(LINT_DEP_INSTALL)
# Combined linting, isomorphic to test-linting.yml's lint job so a local pass means a
# green CI lint: it installs the same env (proxy-dev + generated Prisma client) and then
# runs the diff-scoped ruff format check, whole-tree ruff check, the strict-rule /
# type-discipline / basedpyright budgets as a delta vs the base, then the circular-import
# type-discipline / basedpyright gates as a delta vs the base, then the circular-import
# and import-safety checks. Steps that compare against the base resolve it the same way CI
# does (merge-base with origin's current default branch). Setup (env sync, Prisma client,
# base fetch) runs once up front; the checks themselves are independent, so a sub-make

2
README.md
View file

@ -401,7 +401,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
| [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |
| [Volcengine (`volcengine`)](https://docs.litellm.ai/docs/providers/volcano) | ✅ | ✅ | ✅ | | | | | | | |
| [Voyage AI (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | |
| [VoyageAI by MongoDB (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | |
| [WandB Inference (`wandb`)](https://docs.litellm.ai/docs/providers/wandb_inference) | ✅ | ✅ | ✅ | | | | | | | |
| [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | |
| [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | |

1
backend/routes/allowlist.py
View file

@ -23,6 +23,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/end_user/",
"/sso/",
"/liteadmin/slack/connect/",
"/moyai/connect/",
"/login",
"/v2/login",
"/v3/login",

146
basedpyright-code-budget.json
View file

@ -1,146 +0,0 @@
{
"reportAny": {
"limit": 13429
},
"reportArgumentType": {
"limit": 2198
},
"reportAssignmentType": {
"limit": 319
},
"reportAttributeAccessIssue": {
"limit": 480
},
"reportCallIssue": {
"limit": 112
},
"reportConstantRedefinition": {
"limit": 40
},
"reportDeprecated": {
"limit": 209
},
"reportDuplicateImport": {
"limit": 19
},
"reportExplicitAny": {
"limit": 3369
},
"reportFunctionMemberAccess": {
"limit": 7
},
"reportGeneralTypeIssues": {
"limit": 101
},
"reportIncompatibleMethodOverride": {
"limit": 56
},
"reportIncompatibleVariableOverride": {
"limit": 8
},
"reportInconsistentOverload": {
"limit": 12
},
"reportIndexIssue": {
"limit": 24
},
"reportInvalidTypeForm": {
"limit": 30
},
"reportInvalidTypeVarUse": {
"limit": 1
},
"reportMatchNotExhaustive": {
"limit": 0
},
"reportMissingParameterType": {
"limit": 5570
},
"reportMissingTypeArgument": {
"limit": 15281
},
"reportMissingTypeStubs": {
"limit": 40
},
"reportOperatorIssue": {
"limit": 0
},
"reportOptionalCall": {
"limit": 0
},
"reportOptionalIterable": {
"limit": 0
},
"reportOptionalMemberAccess": {
"limit": 0
},
"reportOptionalOperand": {
"limit": 0
},
"reportOptionalSubscript": {
"limit": 0
},
"reportPossiblyUnboundVariable": {
"limit": 56
},
"reportPrivateUsage": {
"limit": 1804
},
"reportRedeclaration": {
"limit": 8
},
"reportReturnType": {
"limit": 180
},
"reportTypedDictNotRequiredAccess": {
"limit": 22
},
"reportUndefinedVariable": {
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44802
},
"reportUnknownLambdaType": {
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38269
},
"reportUnknownParameterType": {
"limit": 19584
},
"reportUnknownVariableType": {
"limit": 29814
},
"reportUnnecessaryCast": {
"limit": 110
},
"reportUnnecessaryComparison": {
"limit": 687
},
"reportUnnecessaryContains": {
"limit": 4
},
"reportUnnecessaryIsInstance": {
"limit": 816
},
"reportUntypedBaseClass": {
"limit": 0
},
"reportUntypedFunctionDecorator": {
"limit": 27
},
"reportUnusedClass": {
"limit": 21
},
"reportUnusedFunction": {
"limit": 136
},
"reportUnusedImport": {
"limit": 542
},
"reportUnusedVariable": {
"limit": 137
}
}

127
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json
View file

@ -6436,6 +6436,133 @@
],
"title": "litellm_zero_cost_requests rate",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 438
},
"id": 112,
"panels": [],
"title": "Project model rate limits",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Configured rate limit for the Project on the requested model in the current window, by rate_limit_type",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never",
"spanNulls": false
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 439
},
"id": 113,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (project_id, project_alias, requested_model, rate_limit_type) (litellm_project_model_rate_limit_allowed_metric)",
"legendFormat": "{{project_alias}} ({{project_id}}) / {{requested_model}} / {{rate_limit_type}}",
"range": true,
"refId": "A"
}
],
"title": "litellm_project_model_rate_limit_allowed_metric",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Requests or tokens the Project has consumed on the requested model in the current rate limit window, by rate_limit_type",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never",
"spanNulls": false
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 439
},
"id": 114,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (project_id, project_alias, requested_model, rate_limit_type) (litellm_project_model_rate_limit_used_metric)",
"legendFormat": "{{project_alias}} ({{project_id}}) / {{requested_model}} / {{rate_limit_type}}",
"range": true,
"refId": "A"
}
],
"title": "litellm_project_model_rate_limit_used_metric",
"type": "timeseries"
}
],
"preload": false,

59
deploy/lens/README.md
View file

@ -6,21 +6,20 @@ Agent exporters send traces directly to Lens. LiteLLM sends its optional request
## New local installation
Install Docker with Compose and Git, then build the gateway and Lens from one checkout:
Install Docker with Compose, Python 3.10 or later, and Git. Clone LiteLLM, select a published release that includes Lens, and start the existing Compose stack:
```bash
git clone https://github.com/BerriAI/litellm.git
cd litellm
export LITELLM_RELEASE_TAG="sha-$(git rev-parse HEAD)"
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 24)"
export LITELLM_LENS_SERVICE_TOKEN="$(openssl rand -hex 32)"
export OPENAI_API_KEY='<your-provider-key>'
docker compose -f docker/docker-compose.tracing.yml up -d --build
python3 deploy/lens/configure.py --version <release-version>
docker compose --env-file deploy/lens/.env -f deploy/lens/stack.yaml up -d --wait
```
Save the generated keys privately and reuse them when restarting or upgrading. This stack binds to localhost and uses development database passwords; use your normal secrets, TLS, backups, and ingress for a hosted deployment
The configuration command generates your keys and database passwords once, saves them in `deploy/lens/.env` with owner-only permissions, and preserves them on subsequent runs. Back up this file alongside your database volumes. Both images use the selected release; there is no local image build
Open `http://localhost:4002/ui/` and sign in as `admin` with `LITELLM_MASTER_KEY`. Under **Lens > Traces > Set up tracing**, generate a tracing key and copy the ingestion URL. Local exporters use `http://localhost:4318`. Model calls keep their existing LiteLLM URL and model key
Open `http://localhost:4000/ui/` and sign in as `admin` using `LITELLM_MASTER_KEY` from the saved file. Open **Lens**, select your framework, generate a tracing key, and copy the displayed configuration. The trace endpoint is already filled in. Keep your agent's existing model credentials; the tracing key only authorizes trace uploads
PostgreSQL and ClickHouse use persistent Docker volumes and have no host ports. The dashboard and trace listener bind to localhost. Use your normal TLS and ingress for a hosted deployment. Stop the stack with `docker compose --env-file deploy/lens/.env -f deploy/lens/stack.yaml down`; omit `-v` to retain data
Under **Lens > Investigations > Connect worker**, choose an analysis model and monthly budget. The deployed service connects automatically after you save these settings. There is no worker command or second token to copy
@ -68,14 +67,30 @@ Lens does not need provider credentials, PostgreSQL credentials, a GPU, or the L
### Kubernetes with Helm
Both `helm/litellm` and `helm/litellm-helm` support the Lens service. Keep your existing release, namespace, values, and database configuration. Create two Secrets through your normal secret manager: `litellm-lens-service` with key `service-token`, and `litellm-lens-clickhouse` with key `url`
Both `helm/litellm` and `helm/litellm-helm` support Lens. Keep your existing chart, release name, namespace, and values. Add:
```yaml
lensWorker:
enabled: true
```
Then run your usual Helm deployment command using the matching published chart. The chart supplies the matching Lens image, generates the shared service secret, starts a single ClickHouse instance with a persistent volume, and connects the services. Your cluster needs a default storage class, or set `lensWorker.clickhouse.storageClassName`. Bundled storage defaults to 20 GiB; set `lensWorker.clickhouse.storage` before installation to choose another size
When your chart manages an ingress with one hostname, the chart fills in the public tracing address and routes `/lens-ingest` directly to Lens. TLS is detected from `ingress.tls` or an ALB certificate annotation. With custom ingress, multiple hostnames, or TLS terminated elsewhere, set the address explicitly:
```yaml
lensWorker:
enabled: true
publicUrl: https://<your-litellm-host>/lens-ingest
```
For a dedicated trace hostname, configure `lensWorker.ingress.enabled`, `host`, `className`, and `tls`. Its hostname supplies the public address unless you override `publicUrl`. Internal Lens routes stay private
To use an existing ClickHouse database and secrets managed by your platform, keep these overrides:
```yaml
lensWorker:
enabled: true
image:
repository: <matching-worker-image-repository>
digest: sha256:<matching-worker-image-digest>
serviceTokenSecret:
name: litellm-lens-service
key: service-token
@ -84,21 +99,13 @@ lensWorker:
key: url
clickhouseDatabase: litellm
retentionDays: 14
publicUrl: https://<your-litellm-host>/lens-ingest
```
Set `clickhouseDatabase` and `retentionDays` to your existing database and retention before upgrading
Supplying `clickhouseSecret.name` uses that database and disables bundled storage. Keep your database name and retention policy. For GitOps tools that render Helm without cluster access, supply both existing secrets so rendering cannot regenerate credentials
When the chart's main ingress is enabled, it routes `/lens-ingest` directly to Lens. With a custom ingress, add that route yourself. For a dedicated hostname, use `lensWorker.ingress.enabled`, `host`, `className`, and `tls`, and set `publicUrl` to that hostname. The chart connects LiteLLM to Lens internally and gives both services the shared secret
Normal Helm upgrades reuse the generated credentials. Secrets are retained on uninstall, and the ClickHouse volume is retained by Kubernetes. Back them up together. Treat changing the database, storage class, or secret reference as an infrastructure change, not a routine version update
Update your existing component image overrides to matching builds, then use the chart from that checkout:
```bash
helm upgrade --install litellm ./helm/litellm \
--namespace litellm -f values.yaml --wait
```
Use `./helm/litellm-helm` if that is your existing chart. `lensWorker.replicaCount` scales ingestion and investigations. Each replica needs access to the same ClickHouse and gateway. Credentials refresh every 30 seconds; a newly created key may briefly receive a retryable 429. Revocations propagate on refresh, and a replica stops accepting traces when its credential snapshot reaches 90 seconds
After deployment, open **Lens**. If it was already open, click **Check setup**. The setup section moves to your framework and tracing key when Lens is reachable and storage is ready. Investigation setup asks for the analysis model and budget; the installed service connects automatically
## Upgrade
@ -106,13 +113,9 @@ Upgrade LiteLLM and Lens from the same source commit and release identity. For a
Keep the same databases, encryption keys, shared service secret, and public ingestion URL. Pause scheduled investigations and finish or cancel active runs, update both images through your usual deployment process, then check ingestion and run an investigation before resuming schedules. Do not run `docker compose down -v`
When upgrading from the Python worker, replace it with the Rust Lens service, move the existing ClickHouse connection to Lens, and configure the service URLs and secret on LiteLLM. Existing trace data remains in the same ClickHouse database; findings and settings remain in PostgreSQL. Stop the old worker. Generate dedicated tracing keys and change agent exporters to the ingestion URL. A virtual model key no longer authorizes uploads; the old gateway upload endpoints return 410 with setup guidance
If you retain an explicit `LENS_WORKER_TOKEN`, it remains an optional investigation credential. Normal setup uses the shared service connection and registers one managed worker identity. Configure the analysis model and billing key in the dashboard; provider keys stay on LiteLLM
## Development
`make lens-dev` starts LiteLLM, the Rust Lens service, and the hot-reload dashboard. Set `LENS_DEV_PROXY_PORT` and `LENS_DEV_UI_PORT` to change the local ports. For containers, pass the same release identity to both builds. Unversioned or incompatible workers are refused before claiming work
`make lens-dev` starts LiteLLM, Lens, and the hot-reload dashboard. Set `LENS_DEV_PROXY_PORT` and `LENS_DEV_UI_PORT` to change the local ports. For containers, pass the same release identity to both builds. Unversioned or incompatible workers are refused before claiming work
## Configure a lens

2
deploy/lens/config.yaml
View file

@ -1,3 +1,5 @@
model_list: []
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
tracing:

73
deploy/lens/configure.py Normal file
View file

@ -0,0 +1,73 @@
from __future__ import annotations
import argparse
import os
import re
import secrets
import shlex
import sys
from pathlib import Path
from typing import Final
SECRET_NAMES: Final = (
"LITELLM_MASTER_KEY",
"LITELLM_SALT_KEY",
"LITELLM_LENS_SERVICE_TOKEN",
"POSTGRES_PASSWORD",
"CLICKHOUSE_PASSWORD",
)
def environment_content(path: Path) -> str:
if not path.exists():
return "".join(
f"{name}={'sk-' if name.endswith('KEY') else ''}{secrets.token_hex(32)}\n" for name in SECRET_NAMES
)
saved: Final = path.read_text().splitlines()
values: Final = dict(line.split("=", 1) for line in saved if "=" in line)
if any(not values.get(name) for name in SECRET_NAMES):
raise ValueError(f"{path} is incomplete. Restore your saved credentials before continuing")
return "\n".join(line for line in saved if not line.startswith("LITELLM_VERSION=")) + "\n"
def configure(path: Path, version: str) -> None:
release: Final = version.removeprefix("v")
if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:[-.][a-zA-Z0-9.-]+)?", release):
raise ValueError("Use a published release version, such as 1.82.0 or 1.82.0-nightly")
if path.is_symlink():
raise ValueError(f"Refusing to replace a symlink: {path}")
existing: Final = path.exists()
content: Final = environment_content(path)
temporary: Final = path.with_name(f".{path.name}.{secrets.token_hex(8)}")
descriptor: Final = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
try:
with os.fdopen(descriptor, "w") as output:
output.write(content + f"LITELLM_VERSION={release}\n")
if existing:
os.replace(temporary, path)
else:
os.link(temporary, path)
finally:
temporary.unlink(missing_ok=True)
def main() -> None:
parser: Final = argparse.ArgumentParser(description="Create or update the configuration for the Lens Compose stack")
parser.add_argument("--version", required=True, help="Published LiteLLM release; Lens uses the matching version")
parser.add_argument("--env-file", type=Path, default=Path(__file__).with_name(".env"))
arguments: Final = parser.parse_args()
try:
configure(arguments.env_file, arguments.version)
except (OSError, ValueError) as error:
parser.exit(1, f"Could not configure Lens: {error}\n")
sys.stdout.write(
f"Saved {arguments.env_file}. Existing keys and database passwords are preserved\n"
f"Start with: docker compose --env-file {shlex.quote(str(arguments.env_file))} "
"-f deploy/lens/stack.yaml up -d --wait\n"
"Open http://localhost:4000/ui/ and sign in as admin with LITELLM_MASTER_KEY from the saved file\n"
"Back up this file with your database volumes. Do not commit it\n"
)
if __name__ == "__main__":
main()

52
deploy/lens/test_configure.py Normal file
View file

@ -0,0 +1,52 @@
import tempfile
import unittest
from pathlib import Path
from configure import SECRET_NAMES, configure
class ComposeConfigurationTests(unittest.TestCase):
def test_restart_and_upgrade_preserve_private_credentials_and_custom_settings(self) -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / ".env"
configure(path, "v1.2.3")
original = dict(line.split("=", 1) for line in path.read_text().splitlines())
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
self.assertEqual(len({original[name] for name in SECRET_NAMES}), len(SECRET_NAMES))
self.assertTrue(all(len(original[name]) >= 64 for name in SECRET_NAMES))
with path.open("a") as output:
output.write("LITELLM_LENS_PUBLIC_URL=https://traces.example/prefix\n")
configure(path, "1.2.3")
configure(path, "v1.2.4-nightly")
updated = dict(line.split("=", 1) for line in path.read_text().splitlines())
self.assertEqual({name: updated[name] for name in SECRET_NAMES}, {name: original[name] for name in SECRET_NAMES})
self.assertEqual(updated["LITELLM_VERSION"], "1.2.4-nightly")
self.assertEqual(updated["LITELLM_LENS_PUBLIC_URL"], "https://traces.example/prefix")
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
def test_incomplete_configuration_is_never_replaced_with_new_database_passwords(self) -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / ".env"
original = "POSTGRES_PASSWORD=existing\n"
path.write_text(original)
with self.assertRaisesRegex(ValueError, "incomplete"):
configure(path, "1.2.3")
self.assertEqual(path.read_text(), original)
def test_invalid_release_and_symlink_leave_existing_files_untouched(self) -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / ".env"
target = Path(directory) / "saved"
target.write_text("preserve")
path.symlink_to(target)
with self.assertRaisesRegex(ValueError, "symlink"):
configure(path, "1.2.3")
self.assertEqual(target.read_text(), "preserve")
path.unlink()
with self.assertRaisesRegex(ValueError, "published release"):
configure(path, "1.2.3\nPOSTGRES_PASSWORD=replaced")
self.assertFalse(path.exists())
if __name__ == "__main__":
unittest.main()

57
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py
View file

@ -6,7 +6,7 @@ same route are non-inference and free.
"""
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Dict, Final, Optional, Protocol, cast
from typing import TYPE_CHECKING, Dict, Final, Protocol, cast
import litellm
from litellm._logging import verbose_proxy_logger
@ -47,6 +47,10 @@ def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_Manag
return ManagedObjectRepository(prisma_client).table
def _is_response_gone_at_provider(error: Exception, provider_response_id: str) -> bool:
return getattr(error, "status_code", None) == 404 and provider_response_id in str(error)
class CheckResponsesCost:
def __init__(
self,
@ -61,10 +65,15 @@ class CheckResponsesCost:
self.prisma_client: PrismaClient = prisma_client
self.llm_router: Router = llm_router
def _resolve_deployment(self, response_id: str) -> bool:
model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
return model_id is not None and self.llm_router.get_deployment(model_id=model_id) is not None
async def _get_response(
self,
response_id: str,
litellm_metadata: Dict[str, str],
via_router: bool,
) -> ResponsesAPIResponse:
"""Fetch the upstream response, using deployment credentials when available.
@ -75,8 +84,7 @@ class CheckResponsesCost:
sees provider env vars, so it fails for every deployment whose credentials
live in the config; the row then never leaves ``queued``.
"""
model_id: Optional[str] = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
if model_id is None or self.llm_router.get_deployment(model_id=model_id) is None:
if not via_router:
return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
router_response = await self.llm_router.aget_responses(
response_id=response_id, litellm_metadata=litellm_metadata
@ -140,6 +148,8 @@ class CheckResponsesCost:
- Cost is tracked by the get-responses call, billed because the poll is stamped
with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
- Mark responses in a terminal state as complete in the database
- Mark responses the provider no longer has (404 through a resolved
deployment) as stale_expired
"""
try:
await self._cleanup_stale_managed_objects()
@ -159,6 +169,7 @@ class CheckResponsesCost:
verbose_proxy_logger.debug(f"Found {len(jobs)} response jobs to check")
completed_jobs: Final[list[_ManagedObjectRow]] = []
expired_jobs: Final[list[_ManagedObjectRow]] = []
for job in jobs:
unified_object_id = job.unified_object_id
@ -171,31 +182,48 @@ class CheckResponsesCost:
# Get the stored response object to extract model information
stored_response = job.file_object
model_name = stored_response.get("model", None)
# Decrypt the response ID
responses_id_security, _, _ = ResponsesIDSecurity()._decrypt_response_id(unified_object_id)
# Prepare metadata with model information for cost tracking
litellm_metadata = {
"user_api_key_user_id": job.created_by or "default-user-id",
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
}
# Add model information if available
if model_name:
litellm_metadata["model"] = model_name
litellm_metadata["model_group"] = model_name # Use same value for model_group
via_router = self._resolve_deployment(responses_id_security)
except Exception as e:
verbose_proxy_logger.warning(
f"Skipping job {unified_object_id} due to error: {e}"
)
continue
provider_response_id = ResponsesAPIRequestUtils.decode_responses_api_response_id(
responses_id_security
).get("response_id", responses_id_security)
try:
response = await self._get_response(
response_id=responses_id_security,
litellm_metadata=litellm_metadata,
via_router=via_router,
)
verbose_proxy_logger.debug(
f"Response {unified_object_id} status: {response.status}, model: {model_name}"
)
except Exception as e:
if via_router and _is_response_gone_at_provider(e, provider_response_id):
verbose_proxy_logger.info(
f"Response {unified_object_id} no longer available at provider (404), marking stale_expired: {e}"
)
expired_jobs.append(job)
continue
verbose_proxy_logger.warning(
f"Skipping job {unified_object_id} due to error: {e}"
)
@ -210,6 +238,7 @@ class CheckResponsesCost:
# Mark completed jobs in the database
if len(completed_jobs) > 0:
await _managed_object_table(self.prisma_client).update_many(
# bounded-ok: at most MAX_OBJECTS_PER_POLL_CYCLE rows per cycle, the find_many take above
where={"id": {"in": [job.id for job in completed_jobs]}},
data={"status": "completed"},
)
@ -217,3 +246,13 @@ class CheckResponsesCost:
f"Marked {len(completed_jobs)} response jobs as completed"
)
if len(expired_jobs) > 0:
await _managed_object_table(self.prisma_client).update_many(
# bounded-ok: at most MAX_OBJECTS_PER_POLL_CYCLE rows per cycle, the find_many take above
where={"id": {"in": [job.id for job in expired_jobs]}},
data={"status": "stale_expired"},
)
verbose_proxy_logger.info(
f"Marked {len(expired_jobs)} response jobs as stale_expired"
)

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.74"
version = "0.1.75"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.74"
version = "0.1.75"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

2
gateway/routes/allowlist.py
View file

@ -62,6 +62,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/rerank",
"/v1/decisions",
"/decisions",
"/v1/systemone",
"/systemone",
"/v1/ocr",
"/ocr",
"/v1/rag/",

27
helm/litellm-helm/templates/_helpers.tpl
View file

@ -371,3 +371,30 @@ shutdown drain window.
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and .Values.lensWorker.enabled .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}

4
helm/litellm-helm/templates/deployment.yaml
View file

@ -60,11 +60,11 @@ spec:
- name: LITELLM_LENS_URL
value: {{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ required "lensWorker.publicUrl is required" .Values.lensWorker.publicUrl | quote }}
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.serviceTokenSecret.name is required" .Values.lensWorker.serviceTokenSecret.name | quote }}
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
{{- end }}
{{- include "litellm.proxyEnv" . | nindent 12 }}

98
helm/litellm-helm/templates/lens/clickhouse.yaml Normal file
View file

@ -0,0 +1,98 @@
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
{{- $name := include "litellm.lensWorker.clickhouseName" . }}
apiVersion: v1
kind: Service
metadata:
name: {{ $name }}
spec:
clusterIP: None
selector:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
ports:
- name: http
port: 8123
targetPort: http
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ $name }}
spec:
serviceName: {{ $name }}
replicas: 1
selector:
matchLabels:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
template:
metadata:
labels:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
spec:
automountServiceAccountToken: false
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
runAsNonRoot: true
runAsUser: 101
runAsGroup: 101
fsGroup: 101
seccompProfile:
type: RuntimeDefault
containers:
- name: clickhouse
image: {{ .Values.lensWorker.clickhouse.image | quote }}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
env:
- name: CLICKHOUSE_USER
value: default
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ $name }}
key: password
- name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT
value: "1"
ports:
- name: http
containerPort: 8123
startupProbe:
httpGet:
path: /ping
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
readinessProbe:
httpGet:
path: /ping
port: http
livenessProbe:
httpGet:
path: /ping
port: http
timeoutSeconds: 3
resources:
{{- toYaml .Values.lensWorker.clickhouse.resources | nindent 12 }}
volumeMounts:
- name: data
mountPath: /var/lib/clickhouse
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [ReadWriteOnce]
{{- if ne .Values.lensWorker.clickhouse.storageClassName nil }}
storageClassName: {{ .Values.lensWorker.clickhouse.storageClassName | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.lensWorker.clickhouse.storage | quote }}
{{- end }}

12
helm/litellm-helm/templates/lens/deployment.yaml
View file

@ -45,13 +45,23 @@ spec:
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.serviceTokenSecret.name is required" .Values.lensWorker.serviceTokenSecret.name | quote }}
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
- name: CLICKHOUSE_HOST
value: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
key: password
{{- else }}
- name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.clickhouseSecret.name is required" .Values.lensWorker.clickhouseSecret.name | quote }}
key: {{ .Values.lensWorker.clickhouseSecret.key | quote }}
{{- end }}
- name: CLICKHOUSE_DATABASE
value: {{ .Values.lensWorker.clickhouseDatabase | quote }}
- name: AGENT_TRACING_RETENTION_DAYS

27
helm/litellm-helm/templates/lens/secrets.yaml Normal file
View file

@ -0,0 +1,27 @@
{{- if and .Values.lensWorker.enabled (not .Values.lensWorker.serviceTokenSecret.name) }}
{{- $name := include "litellm.lensWorker.serviceTokenSecretName" . }}
{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
apiVersion: v1
kind: Secret
metadata:
name: {{ $name }}
annotations:
helm.sh/resource-policy: keep
type: Opaque
data:
{{ .Values.lensWorker.serviceTokenSecret.key }}: {{ if $existing }}{{ required "Saved Lens service secret is missing its key" (index $existing.data .Values.lensWorker.serviceTokenSecret.key) | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
{{- end }}
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
{{- $name := include "litellm.lensWorker.clickhouseName" . }}
{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $name }}
annotations:
helm.sh/resource-policy: keep
type: Opaque
data:
password: {{ if $existing }}{{ required "Saved Lens ClickHouse secret is missing its password" (index $existing.data "password") | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
{{- end }}

84
helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml Normal file
View file

@ -0,0 +1,84 @@
suite: Lens endpoint defaults
templates:
- deployment.yaml
- configmap-litellm.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest

45
helm/litellm-helm/tests/lens_saved_secrets_tests.yaml Normal file
View file

@ -0,0 +1,45 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=

2
helm/litellm-helm/tests/lens_service_tests.yaml
View file

@ -117,7 +117,7 @@ tests:
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in deployment.yaml
template: deployment.yaml
asserts:

160
helm/litellm-helm/tests/lens_setup_tests.yaml Normal file
View file

@ -0,0 +1,160 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates both private credentials for a new installation
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 2
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: lensWorker.clickhouseSecret.name is required
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60

11
helm/litellm-helm/values.yaml
View file

@ -667,6 +667,17 @@ lensWorker:
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:

30
helm/litellm/templates/_helpers.tpl
View file

@ -520,11 +520,11 @@ shutdown drain window.
- name: LITELLM_LENS_URL
value: {{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ required "lensWorker.publicUrl is required" .Values.lensWorker.publicUrl | quote }}
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.serviceTokenSecret.name is required" .Values.lensWorker.serviceTokenSecret.name | quote }}
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
{{- end }}
{{- end -}}
@ -534,3 +534,29 @@ shutdown drain window.
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and .Values.lensWorker.enabled .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}

98
helm/litellm/templates/lens/clickhouse.yaml Normal file
View file

@ -0,0 +1,98 @@
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
{{- $name := include "litellm.lensWorker.clickhouseName" . }}
apiVersion: v1
kind: Service
metadata:
name: {{ $name }}
spec:
clusterIP: None
selector:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
ports:
- name: http
port: 8123
targetPort: http
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ $name }}
spec:
serviceName: {{ $name }}
replicas: 1
selector:
matchLabels:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
template:
metadata:
labels:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: lens-clickhouse
spec:
automountServiceAccountToken: false
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
runAsNonRoot: true
runAsUser: 101
runAsGroup: 101
fsGroup: 101
seccompProfile:
type: RuntimeDefault
containers:
- name: clickhouse
image: {{ .Values.lensWorker.clickhouse.image | quote }}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
env:
- name: CLICKHOUSE_USER
value: default
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ $name }}
key: password
- name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT
value: "1"
ports:
- name: http
containerPort: 8123
startupProbe:
httpGet:
path: /ping
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
readinessProbe:
httpGet:
path: /ping
port: http
livenessProbe:
httpGet:
path: /ping
port: http
timeoutSeconds: 3
resources:
{{- toYaml .Values.lensWorker.clickhouse.resources | nindent 12 }}
volumeMounts:
- name: data
mountPath: /var/lib/clickhouse
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [ReadWriteOnce]
{{- if ne .Values.lensWorker.clickhouse.storageClassName nil }}
storageClassName: {{ .Values.lensWorker.clickhouse.storageClassName | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.lensWorker.clickhouse.storage | quote }}
{{- end }}

12
helm/litellm/templates/lens/deployment.yaml
View file

@ -45,13 +45,23 @@ spec:
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.serviceTokenSecret.name is required" .Values.lensWorker.serviceTokenSecret.name | quote }}
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
- name: CLICKHOUSE_HOST
value: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
- name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.clickhouseName" . | quote }}
key: password
{{- else }}
- name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: {{ required "lensWorker.clickhouseSecret.name is required" .Values.lensWorker.clickhouseSecret.name | quote }}
key: {{ .Values.lensWorker.clickhouseSecret.key | quote }}
{{- end }}
- name: CLICKHOUSE_DATABASE
value: {{ .Values.lensWorker.clickhouseDatabase | quote }}
- name: AGENT_TRACING_RETENTION_DAYS

27
helm/litellm/templates/lens/secrets.yaml Normal file
View file

@ -0,0 +1,27 @@
{{- if and .Values.lensWorker.enabled (not .Values.lensWorker.serviceTokenSecret.name) }}
{{- $name := include "litellm.lensWorker.serviceTokenSecretName" . }}
{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
apiVersion: v1
kind: Secret
metadata:
name: {{ $name }}
annotations:
helm.sh/resource-policy: keep
type: Opaque
data:
{{ .Values.lensWorker.serviceTokenSecret.key }}: {{ if $existing }}{{ required "Saved Lens service secret is missing its key" (index $existing.data .Values.lensWorker.serviceTokenSecret.key) | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
{{- end }}
{{- if include "litellm.lensWorker.bundledClickhouse" . }}
{{- $name := include "litellm.lensWorker.clickhouseName" . }}
{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $name }}
annotations:
helm.sh/resource-policy: keep
type: Opaque
data:
password: {{ if $existing }}{{ required "Saved Lens ClickHouse secret is missing its password" (index $existing.data "password") | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }}
{{- end }}

86
helm/litellm/tests/lens_endpoint_defaults_tests.yaml Normal file
View file

@ -0,0 +1,86 @@
suite: Lens endpoint defaults
templates:
- gateway/deployment.yaml
- gateway/configmap.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.host: gateway.example
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest
values:
- ./values/required.yaml

47
helm/litellm/tests/lens_saved_secrets_tests.yaml Normal file
View file

@ -0,0 +1,47 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
values:
- ./values/required.yaml

2
helm/litellm/tests/lens_service_tests.yaml
View file

@ -136,7 +136,7 @@ tests:
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in gateway/deployment.yaml
template: gateway/deployment.yaml
asserts:

162
helm/litellm/tests/lens_setup_tests.yaml Normal file
View file

@ -0,0 +1,162 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates both private credentials for a new installation
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 2
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: lensWorker.clickhouseSecret.name is required
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
values:
- ./values/required.yaml

13
helm/litellm/tests/lens_worker_tests.yaml
View file

@ -77,13 +77,20 @@ tests:
asserts:
- hasDocuments:
count: 0
- it: requires a shared service secret when enabled
- it: uses the managed service secret when none is supplied
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
asserts:
- failedTemplate:
errorMessage: lensWorker.serviceTokenSecret.name is required
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: RELEASE-NAME-litellm-lens-service
key: service-token
- it: uses the chart release and a secret without granting Kubernetes access
template: lens/deployment.yaml
chart:

11
helm/litellm/values.yaml
View file

@ -644,6 +644,17 @@ lensWorker:
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:

22
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -11,7 +11,7 @@ import time
from collections.abc import Callable
from dataclasses import dataclass, replace
from pathlib import Path
from typing import TYPE_CHECKING, Final, Optional, Union
from typing import TYPE_CHECKING, Final, Optional, Protocol, Union
from urllib.parse import unquote, urlsplit
from litellm_proxy_extras import prisma_toolchain
@ -35,6 +35,12 @@ if TYPE_CHECKING:
import psycopg.sql
class LensCheckConnect(Protocol):
def __call__(
self, conninfo: str, *, connect_timeout: int, autocommit: bool
) -> "psycopg.Connection[tuple[object, ...]]": ...
def str_to_bool(value: Optional[str]) -> bool:
if value is None:
return False
@ -243,11 +249,9 @@ def _redact_credentials(text: str) -> str:
passwords: Final = sorted(_configured_database_passwords(), key=len, reverse=True)
alternation: Final = "|".join(re.escape(password) for password in passwords)
password_pattern: Final = (
re.compile(rf"(?P<lead>:|password=)(?:{alternation})(?=@|&|$|[\s'\"\]),])", re.IGNORECASE)
if passwords
else None
re.compile(rf"(?<![A-Za-z0-9_])(?:{alternation})(?![A-Za-z0-9_])", re.IGNORECASE) if passwords else None
)
result: Final = password_pattern.sub(rf"\g<lead>{_REDACTED}", text) if password_pattern is not None else text
result: Final = password_pattern.sub(_REDACTED, text) if password_pattern is not None else text
return _secret_shape_redactor()(result)
@ -690,7 +694,7 @@ class ProxyExtrasDBManager:
)
@staticmethod
def raise_if_lens_rename_pending() -> None:
def raise_if_lens_rename_pending(connect: LensCheckConnect | None = None) -> None:
database_url: Final = os.environ.get("DATABASE_URL")
if not database_url:
return
@ -698,8 +702,9 @@ class ProxyExtrasDBManager:
import psycopg
except ImportError as exc:
raise RuntimeError("Install psycopg to verify Lens data safety before prisma db push.") from exc
open_connection: Final = connect if connect is not None else psycopg.connect
try:
with psycopg.connect(
with open_connection(
ProxyExtrasDBManager._strip_prisma_query_params(database_url), connect_timeout=10, autocommit=True
) as connection:
legacy: Final = connection.execute(
@ -710,7 +715,8 @@ class ProxyExtrasDBManager:
).fetchone()
except psycopg.Error as exc:
raise RuntimeError(
"Cannot verify Lens data safety; refusing prisma db push. Check database connectivity and psycopg installation."
"Cannot verify Lens data safety; refusing prisma db push. "
f"The database check failed: {_redact_credentials(str(exc)).strip()}"
) from exc
if legacy is not None:
raise RuntimeError(

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.106"
version = "0.4.107"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.106"
version = "0.4.107"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

3
litellm-rust/AGENTS.md
View file

@ -17,6 +17,9 @@ For string-valued enums and their Serde conversions, follow [.agents/skills/rust
Use [`#[rstest]`](https://docs.rs/rstest/latest/rstest/attr.rstest.html) for new and updated tests and [`#[fixture]`](https://docs.rs/rstest/latest/rstest/attr.fixture.html) for reusable setup, injected through typed test arguments. Express input variations as named `#[case::name(...)]` cases instead of loops or duplicated tests so each failure identifies its case. Keep behavior assertions in the test body and fixtures focused on setup. Use the workspace `rstest` dependency
- Never loop over inputs (`for`, `.iter().for_each`, `.all`) inside a test body; give each input its own `#[case::name(...)]`, or use `#[values(...)]` for a cross product
- Exception: a test pinning a Rust table against a repo-owned data file (for example `include_str!` of a JSON config) may iterate that file's entries
## Error definitions
- A crate's errors live in `src/error.rs`, defined with `thiserror`, and re-exported from `lib.rs`

2
litellm-rust/Cargo.lock generated
View file

@ -4248,6 +4248,7 @@ dependencies = [
name = "litellm-llms-types"
version = "0.1.0"
dependencies = [
"indexmap 2.14.0",
"macro_rules_attribute",
"rstest",
"schemars 1.2.2",
@ -6512,6 +6513,7 @@ checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
dependencies = [
"chrono",
"dyn-clone",
"indexmap 2.14.0",
"ref-cast",
"schemars_derive 1.2.2",
"serde",

9
litellm-rust/crates/callbacks-legacy-python/src/adapter.rs
View file

@ -936,9 +936,6 @@ mod payload_tests {
/// The payload phases of `Logging` on top of `StubLogger`, with `pre_call` handing the
/// payload to the case's `on_pre_call`.
const PAYLOAD_LOGGER: &CStr = c"
class Request:
pass
class PayloadLogger(StubLogger):
def update_from_kwargs(self, **update):
self.update = update
@ -954,7 +951,7 @@ class PayloadLogger(StubLogger):
self.record('post_call', None)
self.post = (original_response, api_key, additional_args)
request = Request()
bound = {}
kwargs = {}
logger = PayloadLogger()
on_pre_call = lambda additional_args: None
@ -1225,10 +1222,10 @@ on_pre_call = lambda args: observed.append(
def check():
assert observed == [(True, True)], observed
")]
#[case::request_attribute_behind_an_omitted_keyword(c"
#[case::bound_value_behind_an_omitted_keyword(c"
document = {'type': 'document_url', 'document_url': 'data:application/pdf;base64,YWJj'}
pages = [0]
request.document = document
bound['document'] = document
kwargs = {'pages': pages}
observed = []
on_pre_call = lambda args: observed.append(

44
litellm-rust/crates/callbacks-legacy-python/src/call.rs
View file

@ -13,21 +13,21 @@ use pyo3::{
pub struct PublicCall {
args: Py<PyTuple>,
kwargs: Py<PyDict>,
request: Py<PyAny>,
bound: Py<PyDict>,
}
impl PublicCall {
/// Copies the keyword arguments once, so the legacy path's rewrites never reach the
/// caller's own dict while every value keeps its identity.
pub fn capture(
request: &Bound<'_, PyAny>,
bound: &Bound<'_, PyDict>,
args: &Bound<'_, PyTuple>,
kwargs: &Bound<'_, PyDict>,
) -> PyResult<Self> {
Ok(Self {
args: args.clone().unbind(),
kwargs: kwargs.copy()?.unbind(),
request: request.clone().unbind(),
bound: bound.clone().unbind(),
})
}
@ -55,31 +55,30 @@ impl PublicCall {
py: Python<'py>,
name: &str,
) -> PyResult<Option<Bound<'py, PyAny>>> {
lookup(self.kwargs.bind(py), self.request.bind(py), name)
lookup(self.kwargs.bind(py), self.bound.bind(py), name)
}
pub(crate) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.args)?;
visit.call(&self.kwargs)?;
visit.call(&self.request)
visit.call(&self.bound)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_support::{local, local_dict};
fn capture<'py>(py: Python<'py>, source: &std::ffi::CStr) -> (PublicCall, Bound<'py, PyDict>) {
let locals = PyDict::new(py);
py.run(source, Some(&locals), Some(&locals)).unwrap();
let request = locals.get_item("request").unwrap().unwrap();
let kwargs = locals
.get_item("kwargs")
.unwrap()
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let call = PublicCall::capture(&request, &PyTuple::empty(py), &kwargs).unwrap();
let call = PublicCall::capture(
&local_dict(&locals, "bound"),
&PyTuple::empty(py),
&local_dict(&locals, "kwargs"),
)
.unwrap();
(call, locals)
}
@ -91,25 +90,22 @@ mod tests {
py,
c"
pages = [0]
class Request:
pass
request = Request()
bound = {'pages': [1]}
kwargs = {'pages': pages}
",
);
let caller = locals
.get_item("kwargs")
.unwrap()
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let caller = local_dict(&locals, "kwargs");
call.kwargs()
.bind(py)
.set_item("litellm_call_id", "call")
.unwrap();
assert!(!caller.contains("litellm_call_id").unwrap());
let pages = locals.get_item("pages").unwrap().unwrap();
assert!(call.lookup(py, "pages").unwrap().unwrap().is(&pages));
assert!(
call.lookup(py, "pages")
.unwrap()
.unwrap()
.is(local(&locals, "pages"))
);
});
}
}

24
litellm-rust/crates/callbacks-legacy-python/src/test_support.rs
View file

@ -173,21 +173,23 @@ pub(crate) fn local<'py>(locals: &Bound<'py, PyDict>, name: &str) -> Bound<'py,
locals.get_item(name).unwrap().unwrap()
}
/// A legacy call over the namespace's `kwargs` (or none) and `request` (or `None`).
pub(crate) fn local_dict<'py>(locals: &Bound<'py, PyDict>, name: &str) -> Bound<'py, PyDict> {
local(locals, name).cast_into().unwrap()
}
/// A legacy call over the namespace's `kwargs` and `bound` dicts, each empty when absent.
pub(crate) fn legacy_call(
py: Python<'_>,
locals: &Bound<'_, PyDict>,
asynchronous: bool,
) -> LegacyLogging {
let request = locals
.get_item("request")
.unwrap()
.unwrap_or_else(|| py.None().into_bound(py));
let kwargs = locals
.get_item("kwargs")
.unwrap()
.map(|kwargs| kwargs.cast_into::<PyDict>().unwrap())
.unwrap_or_else(|| PyDict::new(py));
let call = PublicCall::capture(&request, &PyTuple::empty(py), &kwargs).unwrap();
let dict = |name: &str| {
locals
.get_item(name)
.unwrap()
.map(|value| value.cast_into::<PyDict>().unwrap())
.unwrap_or_else(|| PyDict::new(py))
};
let call = PublicCall::capture(&dict("bound"), &PyTuple::empty(py), &dict("kwargs")).unwrap();
LegacyLogging::new(py, crate::LoggingOperation::Ocr, call, asynchronous)
}

121
litellm-rust/crates/host-python/src/argument.rs
View file

@ -2,78 +2,97 @@ use pyo3::{prelude::*, types::PyDict};
pub fn lookup<'py>(
kwargs: &Bound<'py, PyDict>,
request: &Bound<'py, PyAny>,
bound: &Bound<'py, PyDict>,
name: &str,
) -> PyResult<Option<Bound<'py, PyAny>>> {
if let Some(value) = kwargs.get_item(name)? {
return Ok(Some(value));
match kwargs.get_item(name)? {
Some(value) => Ok(Some(value)),
None => bound.get_item(name),
}
if let Ok(bound) = request.cast::<PyDict>() {
return bound.get_item(name);
}
request.getattr_opt(name)
}
pub fn present<'py>(
kwargs: &Bound<'py, PyDict>,
bound: &Bound<'py, PyDict>,
name: &str,
) -> PyResult<Option<Bound<'py, PyAny>>> {
Ok(lookup(kwargs, bound, name)?.filter(|value| !value.is_none()))
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[test]
fn lookup_prefers_the_keyword_even_when_none_and_falls_back_to_the_request() {
fn dicts<'py>(
py: Python<'py>,
kwargs: &str,
bound: &str,
) -> (Bound<'py, PyDict>, Bound<'py, PyDict>) {
let eval = |source: &str| {
py.eval(&std::ffi::CString::new(source).unwrap(), None, None)
.unwrap()
.cast_into::<PyDict>()
.unwrap()
};
(eval(kwargs), eval(bound))
}
#[rstest]
#[case::keyword_wins(
"{'api_key': 'keyword'}",
"{'api_key': 'bound'}",
Some(Some("keyword"))
)]
#[case::explicit_none_wins("{'api_key': None}", "{'api_key': 'bound'}", Some(None))]
#[case::bound_fallback("{}", "{'api_key': 'bound'}", Some(Some("bound")))]
#[case::missing("{}", "{}", None)]
fn lookup_prefers_the_keyword_and_falls_back_to_bound(
#[case] kwargs: &str,
#[case] bound: &str,
#[case] expected: Option<Option<&str>>,
) {
crate::initialize_python();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
key = object()
document = {'type': 'document_url'}
class Request:
api_key = 'from-request'
api_base = 'from-request'
document = document
request = Request()
kwargs = {'api_key': key, 'api_base': None}
",
Some(&locals),
Some(&locals),
)
.unwrap();
let item = |name: &str| locals.get_item(name).unwrap().unwrap();
let kwargs = item("kwargs").cast_into::<PyDict>().unwrap();
let request = item("request");
let find = |name: &str| lookup(&kwargs, &request, name).unwrap();
assert!(find("api_key").unwrap().is(item("key")));
assert!(find("api_base").unwrap().is_none());
assert!(find("document").unwrap().is(item("document")));
assert!(find("model").is_none());
let (kwargs, bound) = dicts(py, kwargs, bound);
let value = lookup(&kwargs, &bound, "api_key")
.unwrap()
.map(|value| value.extract::<Option<String>>().unwrap());
assert_eq!(value, expected.map(|value| value.map(str::to_owned)));
});
}
#[rstest::rstest]
#[case::prepared_value("{'api_key': 'replacement'}", Some("replacement"))]
#[case::explicit_none("{'api_key': None}", None)]
#[case::bound_fallback("{}", Some("original"))]
fn prepared_mapping_overrides_bound_values(
#[case] source: &str,
#[rstest]
#[case::explicit_none_hides_bound("{'api_key': None}", "{'api_key': 'bound'}", None)]
#[case::bound_none("{}", "{'api_key': None}", None)]
#[case::bound_value("{}", "{'api_key': 'bound'}", Some("bound"))]
fn present_treats_none_as_unset(
#[case] kwargs: &str,
#[case] bound: &str,
#[case] expected: Option<&str>,
) {
crate::initialize_python();
Python::attach(|py| {
let (kwargs, bound) = dicts(py, kwargs, bound);
let value = present(&kwargs, &bound, "api_key")
.unwrap()
.map(|value| value.extract::<String>().unwrap());
assert_eq!(value.as_deref(), expected);
});
}
#[rstest]
fn lookup_returns_the_callers_object() {
crate::initialize_python();
Python::attach(|py| {
let document = PyDict::new(py);
let bound = PyDict::new(py);
bound.set_item("api_key", "original").unwrap();
let source = std::ffi::CString::new(source).unwrap();
let prepared = py
.eval(&source, None, None)
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let value = lookup(&prepared, bound.as_any(), "api_key")
bound.set_item("document", &document).unwrap();
let found = lookup(&PyDict::new(py), &bound, "document")
.unwrap()
.unwrap();
assert_eq!(
value.extract::<Option<String>>().unwrap().as_deref(),
expected
);
assert!(found.is(&document));
});
}
}

2
litellm-rust/crates/host-python/src/lib.rs
View file

@ -19,7 +19,7 @@ mod owned;
mod runtime;
mod services;
pub use argument::lookup;
pub use argument::{lookup, present};
pub use binding::PythonBinding;
pub use conversion_cache::{FromPythonCache, ToPythonCache};
pub use driver::{CallOptions, run_call};

20
litellm-rust/crates/lens/src/lib.rs
View file

@ -116,6 +116,7 @@ pub fn router(state: Arc<State>) -> Router {
Router::new()
.route("/internal/read", post(read))
.route("/internal/spend", post(spend))
.route("/internal/feedback", post(feedback))
.route("/internal/credentials", post(credentials))
.route("/internal/status", get(status)),
)
@ -233,6 +234,23 @@ async fn spend(
AppState(state): AppState<Arc<State>>,
headers: HeaderMap,
body: Body,
) -> Result<StatusCode, Error> {
insert(state, headers, body, InsertTable::SpendLogs).await
}
async fn feedback(
AppState(state): AppState<Arc<State>>,
headers: HeaderMap,
body: Body,
) -> Result<StatusCode, Error> {
insert(state, headers, body, InsertTable::LensFeedback).await
}
async fn insert(
state: Arc<State>,
headers: HeaderMap,
body: Body,
table: InsertTable,
) -> Result<StatusCode, Error> {
auth::authorize_service(&headers, &state.service_token)?;
state.require_storage()?;
@ -256,7 +274,7 @@ async fn spend(
&state.storage.client,
state.storage.config.storage().writer(),
state.storage.config.storage().database(),
InsertTable::SpendLogs,
table,
rows,
)
.await?;

176
litellm-rust/crates/lens/tests/receiver.rs
View file

@ -10,7 +10,10 @@ use rstest::rstest;
use serde_json::json;
use sha2::{Digest, Sha256};
use std::{
sync::{Arc, atomic::Ordering},
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
time::Duration,
};
use wiremock::{
@ -116,6 +119,175 @@ async fn agent_picker_query_preserves_scope_through_the_internal_read_route() {
assert_eq!(response.json::<serde_json::Value>().await.unwrap(), result);
}
#[rstest]
#[case::list_paid("list", None, 0.75)]
#[case::list_zero("list", None, 0.0)]
#[case::detail_paid("trace", None, 0.75)]
#[case::paged_zero("trace", Some(1), 0.0)]
#[tokio::test]
async fn internal_reads_refresh_delayed_gateway_amounts(
#[case] operation: &str,
#[case] page_size: Option<u32>,
#[case] cost: f64,
) {
let store = MockServer::start().await;
let start_ms = (unix_seconds() as i64 - 600) * 1000;
let available = Arc::new(AtomicBool::new(false));
Mock::given(body_string_contains("FROM agent_traces_by_key"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": [{
"trace_id": "trace", "trace_ref": "ref", "team_id": "team",
"api_key_hash": "key", "user_id": "owner", "name": "model call",
"service": "agent", "input_preview": "", "status": "STATUS_CODE_OK",
"start_ms": start_ms, "duration_ms": 1, "span_count": 1,
"agent_count": 0, "agent_invocations": 0, "llm_calls": 1,
"tool_calls": 0, "input_tokens": 1, "output_tokens": 1,
"models": ["test-model"], "error_count": 0, "request_ids": []
}]})))
.mount(&store)
.await;
Mock::given(body_string_contains("o.SpanId AS span_id"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": [{
"trace_id": "trace", "span_id": "span", "parent_span_id": "",
"name": "model call", "type": "llm", "agent": "",
"status": "STATUS_CODE_OK", "status_message": "", "error_truncated": 0,
"start_ns": start_ms * 1_000_000, "duration_ns": 1_000_000,
"service": "agent", "input_preview": "", "model": "test-model",
"input_tokens": 1, "output_tokens": 1, "litellm_request_id": "",
"call_keys": ["provider_response:response"], "call_evidence": "complete",
"team_id": "team", "api_key_hash": "key", "user_id": "owner"
}]})))
.expect(2)
.mount(&store)
.await;
let spend_available = available.clone();
Mock::given(body_string_contains("FROM spend_logs FINAL"))
.respond_with(move |_: &wiremock::Request| {
let rows = if spend_available.load(Ordering::Acquire) {
json!([{
"request_id": "request", "litellm_call_id": "",
"response_id": "response", "upstream_response_id": "",
"trace_id": "", "span_id": "", "team_id": "team",
"api_key": "key", "user": "owner", "spend": cost,
"start_ms": start_ms
}])
} else {
json!([])
};
ResponseTemplate::new(200).set_body_json(json!({"data": rows}))
})
.expect(2)
.mount(&store)
.await;
let server = serve(&store.uri(), true).await;
let scope = json!({"all_teams": 0, "user_id": "owner", "team_ids": []});
let (request, summary_path) = if operation == "list" {
(
json!({
"operation": operation, "scope": scope, "start_ms": start_ms,
"end_ms": start_ms + 1000, "cursor": null, "limit": 50
}),
"/data/0",
)
} else {
(
json!({
"operation": operation, "scope": scope, "trace_id": "trace",
"trace_ref": "ref", "cursor": null, "page_size": page_size
}),
"/summary",
)
};
let client = http_client().unwrap();
for expected in [None, Some(cost)] {
if expected.is_some() {
available.store(true, Ordering::Release);
tokio::time::sleep(litellm_traces_cache::LIVE_TTL + Duration::from_millis(200)).await;
}
let response = client
.post(format!("{}/internal/read", server.url))
.bearer_auth(SERVICE_TOKEN)
.json(&request)
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
let body = response.json::<serde_json::Value>().await.unwrap();
let summary = body.pointer(summary_path).unwrap();
assert_eq!(summary["spend"], json!(expected));
assert_eq!(summary["priced_calls"], u64::from(expected.is_some()));
assert_eq!(summary["llm_calls"], 1);
assert!(!body.to_string().contains("gateway_spend_pending"));
}
}
#[rstest]
#[tokio::test]
async fn feedback_summary_query_preserves_scope_through_the_internal_read_route() {
let store = MockServer::start().await;
let result = json!({"data": [{
"trace_id": "1234567890abcdef1234567890abcdef", "trace_ref": "REF",
"count": "2", "average": 5.5, "lowest": "2"
}]});
Mock::given(method("POST"))
.and(body_string_contains("FROM lens_feedback FINAL"))
.and(query_param("param_all_teams", "0"))
.and(query_param("param_team", "feedback-team"))
.and(query_param(
"param_trace_ids",
"['1234567890abcdef1234567890abcdef']",
))
.respond_with(ResponseTemplate::new(200).set_body_json(&result))
.expect(1)
.mount(&store)
.await;
let server = serve(&store.uri(), true).await;
let response = http_client()
.unwrap()
.post(format!("{}/internal/read", server.url))
.bearer_auth(SERVICE_TOKEN)
.json(&json!({
"operation": "query", "name": "feedback_summary", "parameters": {
"all_teams": 0, "team": "feedback-team", "key_hash": "",
"trace_ids": ["1234567890abcdef1234567890abcdef"]
}
}))
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
assert_eq!(response.json::<serde_json::Value>().await.unwrap(), result);
}
#[rstest]
#[tokio::test]
async fn feedback_rows_are_written_to_the_feedback_table() {
let store = MockServer::start().await;
Mock::given(method("POST"))
.and(query_param(
"query",
"INSERT INTO `litellm`.lens_feedback FORMAT JSONEachRow",
))
.respond_with(ResponseTemplate::new(200))
.expect(1)
.mount(&store)
.await;
let server = serve(&store.uri(), true).await;
let response = http_client()
.unwrap()
.post(format!("{}/internal/feedback", server.url))
.bearer_auth(SERVICE_TOKEN)
.json(&json!([{
"TeamId": "team", "ApiKeyHash": "", "TraceId": "1234567890abcdef1234567890abcdef",
"Author": "customer-1042", "Score": 2, "Comment": "wrong command",
"CreatedAt": "2026-10-07T21:57:01.414Z", "UpdatedAt": "2026-10-07T21:57:01.414Z",
"IsDeleted": 0
}]))
.send()
.await
.unwrap();
assert_eq!(response.status(), 204);
}
#[rstest]
#[tokio::test]
async fn ingestion_confirms_storage_and_overwrites_exporter_tenant() {
@ -297,7 +469,7 @@ async fn ingestion_key_cannot_read_or_export_gateway_records() {
let store = MockServer::start().await;
let server = serve(&store.uri(), true).await;
let client = http_client().unwrap();
for path in ["/internal/read", "/internal/spend"] {
for path in ["/internal/read", "/internal/spend", "/internal/feedback"] {
let response = client
.post(format!("{}{path}", server.url))
.bearer_auth(KEY)

6
litellm-rust/crates/llms-types/AGENTS.md
View file

@ -67,3 +67,9 @@ The same ownership rule applies to Messages, Responses, Chat Completions, OCR, a
- Follow the workspace test-placement and `rstest` rules
- Test provider transformations, header policy, and stream execution in their owning crates
- Do not test import locations or Rust source structure as substitutes for behavior
# references
## json_schema.rs
- https://json-schema.org/draft/2020-12/json-schema-core

3
litellm-rust/crates/llms-types/Cargo.toml
View file

@ -6,9 +6,10 @@ license.workspace = true
repository.workspace = true
[features]
schema = ["dep:schemars"]
schema = ["dep:schemars", "schemars/indexmap2"]
[dependencies]
indexmap = { version = "2.14.0", features = ["serde"] }
macro_rules_attribute.workspace = true
schemars = { workspace = true, optional = true }
serde.workspace = true

36
litellm-rust/crates/llms-types/src/formats/AGENTS.md Normal file
View file

@ -0,0 +1,36 @@
# references
These links describe shared format fields and discriminators. The official API reference is authoritative, and SDK sources only clarify shapes the reference leaves implicit. These contracts are partial typed projections, not exhaustive upstream schemas
## chat_completions.rs and chat_completions/content.rs
- https://developers.openai.com/api/reference/resources/chat
SDK wire definitions:
- https://github.com/openai/openai-python/blob/main/src/openai/types/chat/chat_completion_content_part_param.py
## responses/output.rs and responses/streaming_websocket.rs
- https://developers.openai.com/api/reference/resources/responses
- https://developers.openai.com/api/reference/resources/responses/streaming-events
SDK wire definitions:
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response.py
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response_output_item.py
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/mcp_tool_call_error.py
## batches.rs
- https://developers.openai.com/api/reference/resources/batches
## audio_transcription.rs
- https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions
## ocr.rs
- https://docs.mistral.ai/api/endpoint/ocr
Messages references live in `messages/AGENTS.md`. `LiteLLMOcrResponse` follows the Mistral OCR response shape, and `OcrBoundingBox` describes the corner coordinates providers copy into the normalized `bbox`. Normalized `tables` and `keyValuePairs` stay open because providers pass through different native shapes

2
litellm-rust/crates/llms-types/src/formats/audio_transcription.rs
View file

@ -1,6 +1,6 @@
use serde_json::Value;
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AudioTranscriptionResponseData {
pub text: String,
}

6
litellm-rust/crates/llms-types/src/formats/batches.rs
View file

@ -1,4 +1,4 @@
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum BatchStatus {
@ -7,7 +7,7 @@ pub enum BatchStatus {
Completed,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq)]
pub struct BatchRequestCounts {
pub total: u64,
@ -15,7 +15,7 @@ pub struct BatchRequestCounts {
pub failed: u64,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq)]
pub struct BatchResponse {
pub id: String,

34
litellm-rust/crates/llms-types/src/formats/chat_completions.rs
View file

@ -1,8 +1,14 @@
use serde_json::{Map, Value};
use strum::IntoStaticStr;
mod content;
pub use content::{
ChatContentPart, ChatFile, ChatInputAudio, ChatLogprobs, ChatMediaUrl, ChatMediaUrlParameters,
ChatTokenLogprob, ChatTopLogprob, ChatVideoMetadata, PromptCacheBreakpoint, PromptCacheMode,
};
/// Reasoning effort level accepted or applied by the model.
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, IntoStaticStr)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
@ -38,14 +44,14 @@ impl ReasoningEffort {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ChatMessageContent {
Text(String),
Parts(Vec<Value>),
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatMessage {
pub role: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -56,7 +62,7 @@ pub struct ChatMessage {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionToolCallFunctionChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
@ -65,7 +71,7 @@ pub struct ChatCompletionToolCallFunctionChunk {
pub provider_specific_fields: Option<Map<String, Value>>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionToolCallChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
@ -75,7 +81,7 @@ pub struct ChatCompletionToolCallChunk {
pub index: i64,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ChatCompletionThinkingBlock {
Thinking {
@ -96,7 +102,7 @@ pub enum ChatCompletionThinkingBlock {
/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python
/// path reports so cost tracking sees the same numbers on either path.
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct PromptTokensDetails {
pub cached_tokens: u64,
@ -104,7 +110,7 @@ pub struct PromptTokensDetails {
pub text_tokens: u64,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatCompletionsUsage {
pub prompt_tokens: u64,
@ -113,7 +119,7 @@ pub struct ChatCompletionsUsage {
pub prompt_tokens_details: PromptTokensDetails,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsChoiceMessage {
pub role: String,
// Whether an empty turn is `None` or `""` is the provider's choice, not a
@ -123,7 +129,7 @@ pub struct ChatCompletionsChoiceMessage {
pub content: Option<String>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsChoice {
pub index: u64,
pub message: ChatCompletionsChoiceMessage,
@ -135,7 +141,7 @@ pub struct ChatCompletionsChoice {
/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the
/// `ModelResponse` it already created, and echoing the provider's own id here
/// would change it. Pinned by `response_carries_no_id` in the Anthropic chat transformation tests.
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsResponse {
pub created: u64,
pub model: String,
@ -143,7 +149,7 @@ pub struct ChatCompletionsResponse {
pub usage: ChatCompletionsUsage,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatCompletionDelta {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -162,7 +168,7 @@ pub struct ChatCompletionDelta {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionStreamingChoice {
pub index: u64,
pub delta: ChatCompletionDelta,
@ -172,7 +178,7 @@ pub struct ChatCompletionStreamingChoice {
pub logprobs: Option<Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionChunk {
pub id: String,
pub created: u64,

155
litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs Normal file
View file

@ -0,0 +1,155 @@
use serde_json::{Map, Value};
use crate::formats::messages::{CacheControl, CitationsConfig, ContentSource};
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ChatContentPart {
Text {
text: String,
#[serde(skip_serializing_if = "Option::is_none")]
cache_control: Option<CacheControl>,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
ImageUrl {
image_url: ChatMediaUrl,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
VideoUrl {
video_url: ChatMediaUrl,
#[serde(flatten)]
extra: Map<String, Value>,
},
InputAudio {
input_audio: ChatInputAudio,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
File {
file: Box<ChatFile>,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Document {
source: Box<ContentSource>,
#[serde(skip_serializing_if = "Option::is_none")]
title: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
context: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
citations: Option<CitationsConfig>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Refusal {
refusal: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct PromptCacheBreakpoint {
pub mode: PromptCacheMode,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum PromptCacheMode {
Explicit,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ChatMediaUrl {
Url(String),
Parameters(Box<ChatMediaUrlParameters>),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatMediaUrlParameters {
pub url: String,
pub detail: Option<String>,
pub format: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatInputAudio {
pub data: String,
pub format: String,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatFile {
pub file_data: Option<String>,
pub file_id: Option<String>,
pub filename: Option<String>,
pub format: Option<String>,
pub detail: Option<String>,
pub video_metadata: Option<ChatVideoMetadata>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatVideoMetadata {
pub fps: Option<serde_json::Number>,
pub start_offset: Option<String>,
pub end_offset: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatLogprobs {
pub content: Option<Vec<ChatTokenLogprob>>,
pub refusal: Option<Vec<ChatTokenLogprob>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatTokenLogprob {
pub token: String,
pub logprob: serde_json::Number,
pub bytes: Option<Vec<u8>>,
pub top_logprobs: Option<Vec<ChatTopLogprob>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatTopLogprob {
pub token: String,
pub logprob: serde_json::Number,
pub bytes: Option<Vec<u8>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

14
litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md
View file

@ -3,3 +3,17 @@ This directory owns shared Messages API data contracts and serialization: reques
Adapter contracts and execution inputs such as `MessagesTransformContext` belong in `llms/src/base_llm/messages`. Provider rewriting and interpretation belong in `llms/src/<provider>/messages`. Call envelopes, live streams, and call orchestration belong in `inference-messages`
Represent web-search results, encrypted-content fields, and thinking configuration as data here. Decisions to flatten results, remove encrypted content, select thinking budgets, or require beta headers belong to provider transformations. Data-shape validation belongs here, while model capability checks and request adaptation do not
# references
These upstream contracts document fields and discriminators. They are documentation links, not saved golden snapshots. Beta-only blocks and tools (MCP, compaction, tool changes, advisor, fallback, browser state, and toolsets) follow the beta reference
## Messages bodies, content, tools, and streaming
- https://platform.claude.com/docs/en/api/http/messages/create
- https://platform.claude.com/docs/en/api/messages/create.md
- https://platform.claude.com/docs/en/api/beta/messages/create.md
- https://platform.claude.com/docs/en/build-with-claude/streaming.md
- https://platform.claude.com/docs/en/build-with-claude/context-editing
Anthropic-compatible hosts document their deviations in the `llms/src/<provider>/messages` guides. Copy a host reference into `../../providers/AGENTS.md` only when that host gets a typed extension in `providers`

712
litellm-rust/crates/llms-types/src/formats/messages/content.rs Normal file
View file

@ -0,0 +1,712 @@
use serde_json::{Map, Value};
use super::{CacheControl, MessagesToolParam};
use crate::json_schema::JsonSchema;
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ContentSource {
Base64 {
media_type: String,
data: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
Url {
url: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
File {
file_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
Text {
media_type: String,
data: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
Content {
content: BlockContent,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum BlockContent {
Text(String),
Blocks(Vec<MessagesContentPart>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolCaller {
Direct {
#[serde(flatten)]
extra: Map<String, Value>,
},
#[serde(rename = "code_execution_20250825")]
CodeExecution {
tool_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
#[serde(rename = "code_execution_20260120")]
CodeExecution20260120 {
tool_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct CitationsConfig {
pub enabled: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct PageCitation {
pub cited_text: String,
pub document_index: u64,
pub document_title: Option<String>,
pub start_page_number: u64,
pub end_page_number: u64,
pub file_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct CharCitation {
pub cited_text: String,
pub document_index: u64,
pub document_title: Option<String>,
pub start_char_index: u64,
pub end_char_index: u64,
pub file_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ContentBlockCitation {
pub cited_text: String,
pub document_index: u64,
pub document_title: Option<String>,
pub start_block_index: u64,
pub end_block_index: u64,
pub file_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchCitation {
pub cited_text: String,
pub url: String,
pub encrypted_index: String,
pub title: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct SearchResultCitation {
pub cited_text: String,
pub search_result_index: u64,
pub source: String,
pub title: Option<String>,
pub start_block_index: u64,
pub end_block_index: u64,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum Citation {
PageLocation(PageCitation),
CharLocation(CharCitation),
WebSearchResultLocation(WebSearchCitation),
ContentBlockLocation(ContentBlockCitation),
SearchResultLocation(SearchResultCitation),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MessagesContentPart {
Text(TextBlock),
Image(ImageBlock),
Document(Box<DocumentBlock>),
SearchResult(SearchResultBlock),
Thinking(ThinkingBlock),
RedactedThinking(RedactedThinkingBlock),
ToolUse(ToolUseBlock),
ToolResult(ToolResultBlock),
ToolReference(ToolReferenceBlock),
BrowserState(BrowserStateBlock),
ServerToolUse(ServerToolUseBlock),
WebSearchToolResult(ServerToolResultBlock<WebSearchToolResultContent>),
WebFetchToolResult(ServerToolResultBlock<WebFetchToolResultContent>),
CodeExecutionToolResult(ServerToolResultBlock<CodeExecutionToolResultContent>),
BashCodeExecutionToolResult(ServerToolResultBlock<BashCodeExecutionToolResultContent>),
TextEditorCodeExecutionToolResult(
ServerToolResultBlock<TextEditorCodeExecutionToolResultContent>,
),
ToolSearchToolResult(ServerToolResultBlock<ToolSearchToolResultContent>),
AdvisorToolResult(ServerToolResultBlock<AdvisorToolResultContent>),
ContainerUpload(ContainerUploadBlock),
McpToolUse(McpToolUseBlock),
McpToolResult(McpToolResultBlock),
McpToolListing(McpToolListingBlock),
Compaction(CompactionBlock),
ToolAddition(ToolChangeBlock),
ToolRemoval(ToolChangeBlock),
Fallback(FallbackBlock),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct TextBlock {
pub text: String,
pub citations: Option<Vec<Citation>>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ImageBlock {
pub source: ContentSource,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct DocumentBlock {
pub source: ContentSource,
pub title: Option<String>,
pub context: Option<String>,
pub citations: Option<CitationsConfig>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct SearchResultBlock {
pub source: String,
pub title: String,
pub content: Vec<MessagesContentPart>,
pub citations: Option<CitationsConfig>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ThinkingBlock {
pub thinking: String,
pub signature: String,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct RedactedThinkingBlock {
pub data: String,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolUseBlock {
pub id: String,
pub name: String,
pub input: Map<String, Value>,
pub caller: Option<ToolCaller>,
pub toolset_name: Option<String>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolResultBlock {
pub tool_use_id: String,
pub content: Option<BlockContent>,
pub is_error: Option<bool>,
pub toolset_name: Option<String>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolReferenceBlock {
pub tool_name: String,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct BrowserStateBlock {
pub tabs: Vec<BrowserTab>,
pub state_changes: Option<Vec<BrowserStateChange>>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct BrowserTab {
pub tab_id: String,
pub title: String,
pub url: String,
pub active: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum BrowserStateChange {
TabOpened {
tab_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
DownloadStarted {
download_id: String,
url: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
DownloadCompleted {
download_id: String,
url: String,
#[serde(skip_serializing_if = "Option::is_none")]
path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
size_bytes: Option<u64>,
#[serde(flatten)]
extra: Map<String, Value>,
},
DownloadFailed {
download_id: String,
url: String,
#[serde(skip_serializing_if = "Option::is_none")]
error: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ServerToolUseBlock {
pub id: String,
pub name: String,
pub input: Map<String, Value>,
pub caller: Option<ToolCaller>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ServerToolResultBlock<C> {
pub tool_use_id: String,
pub content: C,
pub caller: Option<ToolCaller>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ServerToolError {
pub error_code: String,
pub error_message: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum WebSearchToolResultContent {
Error(WebSearchResultError),
Results(Vec<WebSearchResult>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchResultError {
#[serde(rename = "type")]
pub error_type: WebSearchResultErrorType,
pub error_code: String,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum WebSearchResultErrorType {
WebSearchToolResultError,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchResult {
#[serde(rename = "type")]
pub result_type: WebSearchResultType,
pub url: String,
pub title: String,
pub encrypted_content: String,
pub page_age: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum WebSearchResultType {
WebSearchResult,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum WebFetchToolResultContent {
WebFetchToolResultError(ServerToolError),
WebFetchResult(WebFetchResult),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebFetchResult {
pub url: String,
pub content: WebFetchDocument,
pub retrieved_at: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum WebFetchDocument {
Document(Box<DocumentBlock>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum CodeExecutionToolResultContent {
CodeExecutionToolResultError(ServerToolError),
CodeExecutionResult(CodeExecutionResult),
EncryptedCodeExecutionResult(EncryptedCodeExecutionResult),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum BashCodeExecutionToolResultContent {
BashCodeExecutionToolResultError(ServerToolError),
BashCodeExecutionResult(CodeExecutionResult<BashCodeExecutionOutput>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct CodeExecutionResult<O = CodeExecutionOutput> {
pub stdout: String,
pub stderr: String,
pub return_code: i64,
pub content: Vec<O>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct EncryptedCodeExecutionResult {
pub encrypted_stdout: String,
pub stderr: String,
pub return_code: i64,
pub content: Vec<CodeExecutionOutput>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum CodeExecutionOutput {
CodeExecutionOutput {
file_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum BashCodeExecutionOutput {
BashCodeExecutionOutput {
file_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum TextEditorCodeExecutionToolResultContent {
TextEditorCodeExecutionToolResultError(ServerToolError),
TextEditorCodeExecutionViewResult(TextEditorViewResult),
TextEditorCodeExecutionCreateResult(TextEditorCreateResult),
TextEditorCodeExecutionStrReplaceResult(TextEditorStrReplaceResult),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct TextEditorViewResult {
pub content: String,
pub file_type: TextEditorFileType,
pub num_lines: Option<u64>,
pub start_line: Option<u64>,
pub total_lines: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum TextEditorFileType {
Text,
Image,
Pdf,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct TextEditorCreateResult {
pub is_file_update: bool,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct TextEditorStrReplaceResult {
pub lines: Option<Vec<String>>,
pub new_lines: Option<u64>,
pub new_start: Option<u64>,
pub old_lines: Option<u64>,
pub old_start: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolSearchToolResultContent {
ToolSearchToolResultError(ServerToolError),
ToolSearchToolSearchResult(ToolSearchResult),
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolSearchResult {
pub tool_references: Vec<ToolSearchReference>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolSearchReference {
ToolReference(ToolReferenceBlock),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum AdvisorToolResultContent {
AdvisorToolResultError(ServerToolError),
AdvisorResult {
text: String,
#[serde(skip_serializing_if = "Option::is_none")]
stop_reason: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
AdvisorRedactedResult {
encrypted_content: String,
#[serde(skip_serializing_if = "Option::is_none")]
stop_reason: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ContainerUploadBlock {
pub file_id: String,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpToolUseBlock {
pub id: String,
pub name: String,
pub server_name: String,
pub input: Map<String, Value>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpToolResultBlock {
pub tool_use_id: String,
pub content: Option<McpToolResultContent>,
pub is_error: Option<bool>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum McpToolResultContent {
Text(String),
Blocks(Vec<McpToolResultText>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum McpToolResultText {
Text(TextBlock),
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpToolListingBlock {
pub mcp_server_name: String,
pub tools: Vec<McpListedTool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpListedTool {
pub name: String,
pub description: Option<String>,
pub input_schema: JsonSchema,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct CompactionBlock {
pub content: Option<String>,
pub encrypted_content: Option<String>,
pub signature: Option<String>,
pub tool_changes: Option<Vec<ToolChange>>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolChange {
ToolAddition(ToolChangeBlock),
ToolRemoval(ToolChangeBlock),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolChangeBlock {
pub tool: ToolChangeTarget,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolChangeTarget {
ToolReference {
name: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
McpToolReference {
server_name: String,
name: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
McpToolsetReference {
server_name: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
ToolDefinition {
definition: Box<MessagesToolParam>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct FallbackBlock {
pub from: FallbackModel,
pub to: FallbackModel,
pub trigger: Option<FallbackTrigger>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct FallbackModel {
pub model: String,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum FallbackTrigger {
Refusal {
#[serde(skip_serializing_if = "Option::is_none")]
category: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}

331
litellm-rust/crates/llms-types/src/formats/messages/metadata.rs Normal file
View file

@ -0,0 +1,331 @@
use serde_json::{Map, Value};
use crate::json_schema::JsonSchema;
use crate::recognized::Recognized;
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesMetadata {
pub user_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct OutputFormat {
#[serde(rename = "type")]
pub format_type: OutputFormatType,
pub schema: JsonSchema,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum OutputFormatType {
JsonSchema,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesCompaction {
#[serde(rename = "type")]
pub compaction_type: CompactionType,
pub instructions: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum CompactionType {
Summarize,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesContainer {
pub id: Option<String>,
pub expires_at: Option<String>,
pub skills: Option<Vec<ContainerSkill>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ContainerSkill {
#[serde(rename = "type")]
pub skill_type: SkillType,
pub skill_id: String,
pub version: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum SkillType {
Anthropic,
Custom,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpServer {
#[serde(rename = "type")]
pub server_type: McpServerType,
pub url: String,
pub name: String,
pub authorization_token: Option<String>,
pub tool_configuration: Option<McpToolConfiguration>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum McpServerType {
Url,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct McpToolConfiguration {
pub allowed_tools: Option<Vec<String>>,
pub enabled: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct StopDetails {
#[serde(rename = "type")]
pub detail_type: Recognized<StopDetailsType>,
pub category: Option<String>,
pub explanation: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum StopDetailsType {
Refusal,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ContextManagementResponse {
pub applied_edits: Option<Vec<AppliedEdit>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct AppliedEdit {
#[serde(rename = "type")]
pub edit_type: Option<String>,
pub cleared_input_tokens: Option<u64>,
pub cleared_tool_uses: Option<u64>,
pub cleared_thinking_turns: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct Safeguard {
#[serde(rename = "type")]
pub safeguard_type: String,
pub classifier_context: Option<Map<String, Value>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[derive(
Clone,
Debug,
PartialEq,
Eq,
strum::Display,
strum::EnumString,
strum::AsRefStr,
serde_with::DeserializeFromStr,
serde_with::SerializeDisplay,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]
#[strum(serialize_all = "snake_case")]
pub enum MessageRole {
User,
Assistant,
System,
#[strum(default, transparent)]
Other(String),
}
impl MessageRole {
pub fn as_str(&self) -> &str {
self.as_ref()
}
}
impl From<String> for MessageRole {
fn from(value: String) -> Self {
value.parse().unwrap_or_else(|never| match never {})
}
}
impl From<MessageRole> for String {
fn from(value: MessageRole) -> Self {
value.to_string()
}
}
#[derive(
Clone,
Debug,
PartialEq,
Eq,
strum::Display,
strum::EnumString,
strum::AsRefStr,
serde_with::DeserializeFromStr,
serde_with::SerializeDisplay,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]
#[strum(serialize_all = "snake_case")]
pub enum MessageType {
Message,
#[strum(default, transparent)]
Other(String),
}
impl MessageType {
pub fn as_str(&self) -> &str {
self.as_ref()
}
}
impl From<String> for MessageType {
fn from(value: String) -> Self {
value.parse().unwrap_or_else(|never| match never {})
}
}
impl From<MessageType> for String {
fn from(value: MessageType) -> Self {
value.to_string()
}
}
#[derive(
Clone,
Debug,
PartialEq,
Eq,
strum::Display,
strum::EnumString,
strum::AsRefStr,
serde_with::DeserializeFromStr,
serde_with::SerializeDisplay,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]
#[strum(serialize_all = "snake_case")]
pub enum StopReason {
EndTurn,
MaxTokens,
StopSequence,
ToolUse,
Refusal,
Compaction,
PauseTurn,
ModelContextWindowExceeded,
#[strum(default, transparent)]
Other(String),
}
impl StopReason {
pub fn as_str(&self) -> &str {
self.as_ref()
}
}
impl From<String> for StopReason {
fn from(value: String) -> Self {
value.parse().unwrap_or_else(|never| match never {})
}
}
impl From<StopReason> for String {
fn from(value: StopReason) -> Self {
value.to_string()
}
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ContainerReference {
Id(String),
Parameters(Box<MessagesContainer>),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesDiagnosticsParam {
#[serde(
default,
skip_serializing_if = "Option::is_none",
with = "::serde_with::rust::double_option"
)]
#[cfg_attr(feature = "schema", schemars(with = "Option<String>"))]
pub previous_message_id: Option<Option<String>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesDiagnostics {
pub cache_miss_reason: Option<Recognized<CacheMissReason>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum CacheMissReason {
ModelChanged(CacheMissedTokens),
SystemChanged(CacheMissedTokens),
ToolsChanged(CacheMissedTokens),
MessagesChanged(CacheMissedTokens),
PreviousMessageNotFound {
#[serde(flatten)]
extra: Map<String, Value>,
},
Unavailable {
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct CacheMissedTokens {
pub cache_missed_input_tokens: u64,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

45
litellm-rust/crates/llms-types/src/formats/messages/mod.rs
View file

@ -1,11 +1,54 @@
mod content;
mod metadata;
mod request;
mod response;
pub mod streaming;
mod tools;
mod usage;
pub use content::{
AdvisorToolResultContent, BashCodeExecutionOutput, BashCodeExecutionToolResultContent,
BlockContent, BrowserStateBlock, BrowserStateChange, BrowserTab, CharCitation, Citation,
CitationsConfig, CodeExecutionOutput, CodeExecutionResult, CodeExecutionToolResultContent,
CompactionBlock, ContainerUploadBlock, ContentBlockCitation, ContentSource, DocumentBlock,
EncryptedCodeExecutionResult, FallbackBlock, FallbackModel, FallbackTrigger, ImageBlock,
McpListedTool, McpToolListingBlock, McpToolResultBlock, McpToolResultContent,
McpToolResultText, McpToolUseBlock, MessagesContentPart, PageCitation, RedactedThinkingBlock,
SearchResultBlock, SearchResultCitation, ServerToolError, ServerToolResultBlock,
ServerToolUseBlock, TextBlock, TextEditorCodeExecutionToolResultContent,
TextEditorCreateResult, TextEditorFileType, TextEditorStrReplaceResult, TextEditorViewResult,
ThinkingBlock, ToolCaller, ToolChange, ToolChangeBlock, ToolChangeTarget, ToolReferenceBlock,
ToolResultBlock, ToolSearchReference, ToolSearchResult, ToolSearchToolResultContent,
ToolUseBlock, WebFetchDocument, WebFetchResult, WebFetchToolResultContent, WebSearchCitation,
WebSearchResult, WebSearchResultError, WebSearchResultErrorType, WebSearchResultType,
WebSearchToolResultContent,
};
pub use metadata::{
AppliedEdit, CacheMissReason, CacheMissedTokens, CompactionType, ContainerReference,
ContainerSkill, ContextManagementResponse, McpServer, McpServerType, McpToolConfiguration,
MessageRole, MessageType, MessagesCompaction, MessagesContainer, MessagesDiagnostics,
MessagesDiagnosticsParam, MessagesMetadata, OutputFormat, OutputFormatType, Safeguard,
SkillType, StopDetails, StopDetailsType, StopReason,
};
pub use request::{
AdaptiveThinking, CacheControl, ContentBlock, ContentBlockType, ContextEdit, ContextManagement,
DisabledThinking, EffortLevel, EnabledThinking, Message, MessageContent,
ContextTrigger, DisabledThinking, EffortLevel, EnabledThinking, Message, MessageContent,
MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig, Speed, SystemPrompt,
ThinkingConfig, ThinkingDisplay,
};
pub use response::MessagesResponse;
pub use tools::{
AdvisorTool, AdvisorToolName, AllowedCaller, BashToolName, BrowserToolsetConfigs,
BuiltinMessagesTool, ClientTool, CodeExecutionToolName, ComputerTool, ComputerTool20251124,
ComputerToolName, ComputerToolsetConfigs, CustomTool, CustomToolType, McpToolset,
MemoryToolName, MessagesToolParam, ResponseInclusion, ServerTool, StrReplaceBasedEditToolName,
StrReplaceEditorName, TextEditorTool20250728, ToolChoice, ToolChoiceType, ToolResultUrlSource,
ToolSearchBm25ToolName, ToolSearchRegexToolName, Toolset, ToolsetToolConfig,
UrlSourceToolReference, UserInputUrlSource, UserLocationType, WebFetchTool,
WebFetchTool20260309, WebFetchTool20260318, WebFetchToolName, WebFetchUrlSources,
WebSearchTool, WebSearchTool20260318, WebSearchToolName, WebSearchUserLocation,
};
pub use usage::{
CacheCreationUsage, MessagesOutputTokensDetails, MessagesUsage, ServerToolUsage,
UsageIteration, UsageIterationType,
};

51
litellm-rust/crates/llms-types/src/formats/messages/request.rs
View file

@ -4,14 +4,14 @@ use strum::IntoStaticStr;
use crate::formats::chat_completions::ReasoningEffort;
use crate::recognized::Recognized;
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum SystemPrompt {
Text(String),
Blocks(Vec<ContentBlock>),
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum MessageContent {
Text(String),
@ -57,7 +57,7 @@ impl From<ContentBlockType> for String {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ContentBlock {
#[serde(rename = "type", default, skip_serializing_if = "Option::is_none")]
@ -102,7 +102,7 @@ impl ContentBlock {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct CacheControl {
#[serde(rename = "type", skip_serializing_if = "Option::is_none")]
@ -115,7 +115,7 @@ pub struct CacheControl {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct Message {
pub role: String,
pub content: MessageContent,
@ -123,7 +123,7 @@ pub struct Message {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Hash, IntoStaticStr, Eq)]
#[serde(rename_all = "lowercase")]
#[strum(serialize_all = "lowercase")]
@ -153,7 +153,7 @@ impl From<EffortLevel> for ReasoningEffort {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, IntoStaticStr, Eq)]
#[serde(rename_all = "lowercase")]
#[strum(serialize_all = "lowercase")]
@ -170,7 +170,7 @@ impl Speed {
/// The tools whose presence changes how the request is sent. Every other tool, custom or
/// server, deserializes as `Recognized::Unrecognized` and passes through verbatim.
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type")]
pub enum MessagesTool {
#[serde(rename = "advisor_20260301")]
@ -190,7 +190,22 @@ pub enum MessagesTool {
},
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ContextTrigger {
InputTokens {
value: u64,
#[serde(flatten)]
extra: Map<String, Value>,
},
ToolUses {
value: u64,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type")]
pub enum ContextEdit {
#[serde(rename = "compact_20260112")]
@ -210,7 +225,7 @@ pub enum ContextEdit {
},
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ContextManagement {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -219,7 +234,7 @@ pub struct ContextManagement {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OutputConfig {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -236,7 +251,7 @@ impl OutputConfig {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "lowercase")]
pub enum ThinkingDisplay {
@ -245,7 +260,7 @@ pub enum ThinkingDisplay {
Updates,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct EnabledThinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -256,7 +271,7 @@ pub struct EnabledThinking {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct AdaptiveThinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -265,14 +280,14 @@ pub struct AdaptiveThinking {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct DisabledThinking {
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "lowercase")]
pub enum ThinkingConfig {
Enabled(EnabledThinking),
@ -296,7 +311,7 @@ impl ThinkingConfig {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesRequest {
pub model: String,
pub messages: Vec<Message>,
@ -304,7 +319,7 @@ pub struct MessagesRequest {
pub params: MessagesOptionalParams,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesOptionalParams {
#[serde(skip_serializing_if = "Option::is_none")]

2
litellm-rust/crates/llms-types/src/formats/messages/response.rs
View file

@ -1,6 +1,6 @@
use serde_json::{Map, Value};
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesResponse {
pub id: String,
#[serde(rename = "type")]

14
litellm-rust/crates/llms-types/src/formats/messages/streaming.rs
View file

@ -1,6 +1,6 @@
use serde_json::{Map, Value};
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesStreamUsage {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -17,7 +17,7 @@ pub struct MessagesStreamUsage {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesStreamMessage {
pub id: String,
#[serde(rename = "type")]
@ -32,7 +32,7 @@ pub struct MessagesStreamMessage {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MessagesContentBlockDelta {
TextDelta {
@ -56,7 +56,7 @@ pub enum MessagesContentBlockDelta {
},
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesContentBlock {
#[serde(rename = "type")]
pub block_type: String,
@ -82,7 +82,7 @@ pub struct MessagesContentBlock {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesDelta {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -97,7 +97,7 @@ pub struct MessagesDelta {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesStreamError {
#[serde(rename = "type")]
pub error_type: String,
@ -108,7 +108,7 @@ pub struct MessagesStreamError {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MessagesStreamEvent {
MessageStart {

564
litellm-rust/crates/llms-types/src/formats/messages/tools.rs Normal file
View file

@ -0,0 +1,564 @@
use indexmap::IndexMap;
use serde_json::{Map, Value};
use super::{CacheControl, CitationsConfig, McpListedTool};
use crate::json_schema::JsonSchema;
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
pub enum AllowedCaller {
#[serde(rename = "direct")]
Direct,
#[serde(rename = "code_execution_20250825")]
CodeExecution20250825,
#[serde(rename = "code_execution_20260120")]
CodeExecution20260120,
#[serde(rename = "code_execution_20260521")]
CodeExecution20260521,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchUserLocation {
#[serde(rename = "type")]
pub location_type: UserLocationType,
pub city: Option<String>,
pub country: Option<String>,
pub region: Option<String>,
pub timezone: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum UserLocationType {
Approximate,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolChoice {
#[serde(rename = "type")]
pub choice_type: ToolChoiceType,
pub name: Option<String>,
pub disable_parallel_tool_use: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum ToolChoiceType {
Auto,
Any,
Tool,
None,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct CustomTool {
#[serde(rename = "type")]
pub tool_type: Option<CustomToolType>,
pub name: String,
pub input_schema: JsonSchema,
pub description: Option<String>,
pub strict: Option<bool>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub input_examples: Option<Vec<Map<String, Value>>>,
pub eager_input_streaming: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum CustomToolType {
Custom,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum BashToolName {
Bash,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum StrReplaceEditorName {
StrReplaceEditor,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum StrReplaceBasedEditToolName {
StrReplaceBasedEditTool,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum MemoryToolName {
Memory,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ComputerToolName {
Computer,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum CodeExecutionToolName {
CodeExecution,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ToolSearchRegexToolName {
ToolSearchToolRegex,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ToolSearchBm25ToolName {
ToolSearchToolBm25,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum WebSearchToolName {
WebSearch,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum WebFetchToolName {
WebFetch,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum AdvisorToolName {
Advisor,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ServerTool<N> {
pub name: N,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ClientTool<N> {
pub name: N,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub input_examples: Option<Vec<Map<String, Value>>>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct TextEditorTool20250728 {
pub name: StrReplaceBasedEditToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub input_examples: Option<Vec<Map<String, Value>>>,
pub max_characters: Option<u64>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ComputerTool {
pub name: ComputerToolName,
pub display_width_px: u64,
pub display_height_px: u64,
pub display_number: Option<u64>,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub input_examples: Option<Vec<Map<String, Value>>>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ComputerTool20251124 {
pub name: ComputerToolName,
pub display_width_px: u64,
pub display_height_px: u64,
pub display_number: Option<u64>,
pub enable_zoom: Option<bool>,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub input_examples: Option<Vec<Map<String, Value>>>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ResponseInclusion {
Full,
Excluded,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchTool {
pub name: WebSearchToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub allowed_domains: Option<Vec<String>>,
pub blocked_domains: Option<Vec<String>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub max_uses: Option<u64>,
pub strict: Option<bool>,
pub user_location: Option<WebSearchUserLocation>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchTool20260318 {
pub name: WebSearchToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub allowed_domains: Option<Vec<String>>,
pub blocked_domains: Option<Vec<String>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub max_uses: Option<u64>,
pub response_inclusion: Option<ResponseInclusion>,
pub strict: Option<bool>,
pub user_location: Option<WebSearchUserLocation>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum UrlSourceToolReference {
ToolReference {
name: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolResultUrlSource {
All {
#[serde(flatten)]
extra: Map<String, Value>,
},
None {
#[serde(flatten)]
extra: Map<String, Value>,
},
Only {
tools: Vec<UrlSourceToolReference>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Except {
tools: Vec<UrlSourceToolReference>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum UserInputUrlSource {
All {
#[serde(flatten)]
extra: Map<String, Value>,
},
None {
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebFetchUrlSources {
pub client_tool_results: Option<ToolResultUrlSource>,
pub server_tool_results: Option<ToolResultUrlSource>,
pub user_input: Option<UserInputUrlSource>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebFetchTool {
pub name: WebFetchToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub allowed_domains: Option<Vec<String>>,
pub blocked_domains: Option<Vec<String>>,
pub cache_control: Option<CacheControl>,
pub citations: Option<CitationsConfig>,
pub defer_loading: Option<bool>,
pub max_content_tokens: Option<u64>,
pub max_uses: Option<u64>,
pub strict: Option<bool>,
pub url_sources: Option<WebFetchUrlSources>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebFetchTool20260309 {
pub name: WebFetchToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub allowed_domains: Option<Vec<String>>,
pub blocked_domains: Option<Vec<String>>,
pub cache_control: Option<CacheControl>,
pub citations: Option<CitationsConfig>,
pub defer_loading: Option<bool>,
pub max_content_tokens: Option<u64>,
pub max_uses: Option<u64>,
pub strict: Option<bool>,
pub url_sources: Option<WebFetchUrlSources>,
pub use_cache: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebFetchTool20260318 {
pub name: WebFetchToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub allowed_domains: Option<Vec<String>>,
pub blocked_domains: Option<Vec<String>>,
pub cache_control: Option<CacheControl>,
pub citations: Option<CitationsConfig>,
pub defer_loading: Option<bool>,
pub max_content_tokens: Option<u64>,
pub max_uses: Option<u64>,
pub response_inclusion: Option<ResponseInclusion>,
pub strict: Option<bool>,
pub url_sources: Option<WebFetchUrlSources>,
pub use_cache: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AdvisorTool {
pub name: AdvisorToolName,
pub model: String,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub caching: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub max_tokens: Option<u64>,
pub max_uses: Option<u64>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolsetToolConfig {
pub defer_loading: Option<bool>,
pub enabled: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpToolset {
pub mcp_server_name: String,
pub cache_control: Option<CacheControl>,
pub configs: Option<IndexMap<String, ToolsetToolConfig>>,
pub default_config: Option<ToolsetToolConfig>,
pub tools: Option<Vec<McpListedTool>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct BrowserToolsetConfigs {
#[serde(rename = "type")]
pub type_text: Option<ToolsetToolConfig>,
pub close_tab: Option<ToolsetToolConfig>,
pub double_click: Option<ToolsetToolConfig>,
pub file_upload: Option<ToolsetToolConfig>,
pub find: Option<ToolsetToolConfig>,
pub form_input: Option<ToolsetToolConfig>,
pub get_page_text: Option<ToolsetToolConfig>,
pub hold_key: Option<ToolsetToolConfig>,
pub hover: Option<ToolsetToolConfig>,
pub javascript_exec: Option<ToolsetToolConfig>,
pub key: Option<ToolsetToolConfig>,
pub left_click: Option<ToolsetToolConfig>,
pub left_click_drag: Option<ToolsetToolConfig>,
pub left_mouse_down: Option<ToolsetToolConfig>,
pub left_mouse_up: Option<ToolsetToolConfig>,
pub list_tabs: Option<ToolsetToolConfig>,
pub middle_click: Option<ToolsetToolConfig>,
pub mouse_move: Option<ToolsetToolConfig>,
pub navigate: Option<ToolsetToolConfig>,
pub new_tab: Option<ToolsetToolConfig>,
pub read_console: Option<ToolsetToolConfig>,
pub read_network: Option<ToolsetToolConfig>,
pub read_page: Option<ToolsetToolConfig>,
pub right_click: Option<ToolsetToolConfig>,
pub screenshot: Option<ToolsetToolConfig>,
pub scroll: Option<ToolsetToolConfig>,
pub scroll_to: Option<ToolsetToolConfig>,
pub switch_tab: Option<ToolsetToolConfig>,
pub triple_click: Option<ToolsetToolConfig>,
pub wait: Option<ToolsetToolConfig>,
pub zoom: Option<ToolsetToolConfig>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ComputerToolsetConfigs {
#[serde(rename = "type")]
pub type_text: Option<ToolsetToolConfig>,
pub cursor_position: Option<ToolsetToolConfig>,
pub double_click: Option<ToolsetToolConfig>,
pub hold_key: Option<ToolsetToolConfig>,
pub key: Option<ToolsetToolConfig>,
pub left_click: Option<ToolsetToolConfig>,
pub left_click_drag: Option<ToolsetToolConfig>,
pub left_mouse_down: Option<ToolsetToolConfig>,
pub left_mouse_up: Option<ToolsetToolConfig>,
pub middle_click: Option<ToolsetToolConfig>,
pub mouse_move: Option<ToolsetToolConfig>,
pub right_click: Option<ToolsetToolConfig>,
pub screenshot: Option<ToolsetToolConfig>,
pub scroll: Option<ToolsetToolConfig>,
pub triple_click: Option<ToolsetToolConfig>,
pub wait: Option<ToolsetToolConfig>,
pub zoom: Option<ToolsetToolConfig>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct Toolset<C> {
pub cache_control: Option<CacheControl>,
pub configs: Option<Box<C>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type")]
pub enum BuiltinMessagesTool {
#[serde(rename = "bash_20241022")]
Bash20241022(ClientTool<BashToolName>),
#[serde(rename = "bash_20250124")]
Bash20250124(ClientTool<BashToolName>),
#[serde(rename = "text_editor_20241022")]
TextEditor20241022(ClientTool<StrReplaceEditorName>),
#[serde(rename = "text_editor_20250124")]
TextEditor20250124(ClientTool<StrReplaceEditorName>),
#[serde(rename = "text_editor_20250429")]
TextEditor20250429(ClientTool<StrReplaceBasedEditToolName>),
#[serde(rename = "text_editor_20250728")]
TextEditor20250728(TextEditorTool20250728),
#[serde(rename = "memory_20250818")]
Memory20250818(ClientTool<MemoryToolName>),
#[serde(rename = "computer_20241022")]
Computer20241022(ComputerTool),
#[serde(rename = "computer_20250124")]
Computer20250124(ComputerTool),
#[serde(rename = "computer_20251124")]
Computer20251124(ComputerTool20251124),
#[serde(rename = "code_execution_20250522")]
CodeExecution20250522(ServerTool<CodeExecutionToolName>),
#[serde(rename = "code_execution_20250825")]
CodeExecution20250825(ServerTool<CodeExecutionToolName>),
#[serde(rename = "code_execution_20260120")]
CodeExecution20260120(ServerTool<CodeExecutionToolName>),
#[serde(rename = "code_execution_20260521")]
CodeExecution20260521(ServerTool<CodeExecutionToolName>),
#[serde(rename = "tool_search_tool_regex_20251119")]
ToolSearchRegex20251119(ServerTool<ToolSearchRegexToolName>),
#[serde(rename = "tool_search_tool_regex")]
ToolSearchRegex(ServerTool<ToolSearchRegexToolName>),
#[serde(rename = "tool_search_tool_bm25_20251119")]
ToolSearchBm2520251119(ServerTool<ToolSearchBm25ToolName>),
#[serde(rename = "tool_search_tool_bm25")]
ToolSearchBm25(ServerTool<ToolSearchBm25ToolName>),
#[serde(rename = "web_search_20250305")]
WebSearch20250305(WebSearchTool),
#[serde(rename = "web_search_20260209")]
WebSearch20260209(WebSearchTool),
#[serde(rename = "web_search_20260318")]
WebSearch20260318(WebSearchTool20260318),
#[serde(rename = "web_fetch_20250910")]
WebFetch20250910(WebFetchTool),
#[serde(rename = "web_fetch_20260209")]
WebFetch20260209(WebFetchTool),
#[serde(rename = "web_fetch_20260309")]
WebFetch20260309(WebFetchTool20260309),
#[serde(rename = "web_fetch_20260318")]
WebFetch20260318(WebFetchTool20260318),
#[serde(rename = "advisor_20260301")]
Advisor20260301(AdvisorTool),
#[serde(rename = "browser_toolset_20260801")]
BrowserToolset20260801(Toolset<BrowserToolsetConfigs>),
#[serde(rename = "computer_toolset_20260801")]
ComputerToolset20260801(Toolset<ComputerToolsetConfigs>),
#[serde(rename = "mcp_toolset")]
McpToolset(McpToolset),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum MessagesToolParam {
Builtin(Box<BuiltinMessagesTool>),
Custom(Box<CustomTool>),
}

75
litellm-rust/crates/llms-types/src/formats/messages/usage.rs Normal file
View file

@ -0,0 +1,75 @@
use serde_json::{Map, Value};
use crate::recognized::Recognized;
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ServerToolUsage {
pub web_search_requests: Option<u64>,
pub web_fetch_requests: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct UsageIteration {
#[serde(rename = "type")]
pub iteration_type: Recognized<UsageIterationType>,
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cache_creation_input_tokens: Option<u64>,
pub cache_read_input_tokens: Option<u64>,
pub cache_creation: Option<CacheCreationUsage>,
pub model: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum UsageIterationType {
Compaction,
Message,
AdvisorMessage,
FallbackMessage,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesUsage {
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cache_creation_input_tokens: Option<u64>,
pub cache_read_input_tokens: Option<u64>,
pub server_tool_use: Option<ServerToolUsage>,
pub cache_creation: Option<CacheCreationUsage>,
pub output_tokens_details: Option<MessagesOutputTokensDetails>,
pub service_tier: Option<String>,
pub inference_geo: Option<String>,
pub speed: Option<super::Speed>,
pub iterations: Option<Vec<UsageIteration>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct CacheCreationUsage {
pub ephemeral_1h_input_tokens: Option<u64>,
pub ephemeral_5m_input_tokens: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesOutputTokensDetails {
pub thinking_tokens: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

3
litellm-rust/crates/llms-types/src/formats/mod.rs
View file

@ -1,3 +1,6 @@
//! Wire types for each provider API format (OpenAI Chat Completions,
//! Anthropic Messages, Responses, ...), one submodule per format.
pub mod audio_transcription;
pub mod batches;
pub mod chat_completions;

26
litellm-rust/crates/llms-types/src/formats/ocr.rs
View file

@ -5,7 +5,7 @@ use serde_with::serde_as;
use crate::serde_compat::{FiniteF64, LaxI64};
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type")]
pub enum OcrDocument {
#[serde(rename = "document_url")]
@ -49,7 +49,7 @@ impl OcrDocument {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Default, Eq)]
#[serde(rename_all = "lowercase")]
pub enum OcrResponseFormat {
@ -59,7 +59,7 @@ pub enum OcrResponseFormat {
}
#[serde_as]
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPageDimensions {
#[serde_as(deserialize_as = "Option<LaxI64>")]
@ -70,7 +70,7 @@ pub struct OcrPageDimensions {
pub width: Option<i64>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPageImage {
pub image_base64: Option<String>,
@ -80,7 +80,7 @@ pub struct OcrPageImage {
}
#[serde_as]
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPage {
#[serde_as(deserialize_as = "LaxI64")]
@ -93,7 +93,7 @@ pub struct OcrPage {
}
#[serde_as]
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrUsageInfo {
#[serde_as(deserialize_as = "Option<LaxI64>")]
@ -108,7 +108,7 @@ pub struct OcrUsageInfo {
pub extra_fields: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct LiteLLMOcrResponse {
pub pages: Vec<OcrPage>,
pub model: String,
@ -150,3 +150,15 @@ impl LiteLLMOcrResponse {
fn ocr_object() -> String {
"ocr".into()
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrBoundingBox {
pub top_left_x: Option<serde_json::Number>,
pub top_left_y: Option<serde_json::Number>,
pub bottom_right_x: Option<serde_json::Number>,
pub bottom_right_y: Option<serde_json::Number>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

2
litellm-rust/crates/llms-types/src/formats/responses/mod.rs
View file

@ -1,4 +1,6 @@
mod output;
mod response;
pub mod streaming_websocket;
pub use output::*;
pub use response::ResponsesApiResponse;

831
litellm-rust/crates/llms-types/src/formats/responses/output.rs Normal file
View file

@ -0,0 +1,831 @@
use crate::formats::chat_completions::PromptCacheBreakpoint;
use serde_json::{Map, Value};
use std::collections::BTreeMap;
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesOutputItem {
Message(ResponsesMessage),
FunctionCall(ResponsesFunctionCall),
CustomToolCall(ResponsesCustomToolCall),
Reasoning(ResponsesReasoning),
WebSearchCall(ResponsesWebSearchCall),
FileSearchCall(ResponsesFileSearchCall),
ImageGenerationCall(ResponsesImageGenerationCall),
CodeInterpreterCall(ResponsesCodeInterpreterCall),
McpCall(ResponsesMcpCall),
McpListTools(ResponsesMcpListTools),
FunctionCallOutput(ResponsesFunctionCallOutput),
ComputerCall(ResponsesComputerCall),
ComputerCallOutput(ResponsesComputerCallOutput),
Program(ResponsesProgram),
ProgramOutput(ResponsesProgramOutput),
ToolSearchCall(ResponsesToolSearchCall),
ToolSearchOutput(ResponsesToolSearchOutput),
AdditionalTools(ResponsesAdditionalTools),
Compaction(ResponsesCompaction),
LocalShellCall(ResponsesLocalShellCall),
LocalShellCallOutput(ResponsesLocalShellCallOutput),
ShellCall(ResponsesShellCall),
ShellCallOutput(ResponsesShellCallOutput),
ApplyPatchCall(ResponsesApplyPatchCall),
ApplyPatchCallOutput(ResponsesApplyPatchCallOutput),
McpApprovalRequest(ResponsesMcpApprovalRequest),
McpApprovalResponse(ResponsesMcpApprovalResponse),
CustomToolCallOutput(ResponsesCustomToolCallOutput),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesContentPart {
OutputText {
text: String,
#[serde(skip_serializing_if = "Option::is_none")]
annotations: Option<Vec<ResponsesAnnotation>>,
#[serde(skip_serializing_if = "Option::is_none")]
logprobs: Option<Vec<crate::formats::chat_completions::ChatTokenLogprob>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Refusal {
refusal: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
SummaryText {
text: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
ReasoningText {
text: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesAnnotation {
UrlCitation(ResponsesUrlCitation),
FileCitation(ResponsesFileCitation),
FilePath(ResponsesFileCitation),
ContainerFileCitation(ResponsesFileCitation),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesCodeOutput {
Logs {
logs: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
Image {
url: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMessage {
pub id: Option<String>,
pub status: Option<String>,
pub role: Option<String>,
pub phase: Option<String>,
pub content: Option<Vec<ResponsesContentPart>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesFunctionCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub name: Option<String>,
pub arguments: Option<String>,
pub namespace: Option<String>,
pub r#async: Option<bool>,
pub caller: Option<ResponsesToolCaller>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesCustomToolCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub name: Option<String>,
pub input: Option<String>,
pub namespace: Option<String>,
pub r#async: Option<bool>,
pub caller: Option<ResponsesToolCaller>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesReasoning {
pub id: Option<String>,
pub status: Option<String>,
pub summary: Option<Vec<ResponsesContentPart>>,
pub content: Option<Vec<ResponsesContentPart>>,
pub encrypted_content: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesWebSearchCall {
pub id: Option<String>,
pub status: Option<String>,
pub action: Option<ResponsesWebSearchAction>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesFileSearchCall {
pub id: Option<String>,
pub status: Option<String>,
pub queries: Option<Vec<String>>,
pub results: Option<Vec<ResponsesFileSearchResult>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesFileSearchResult {
pub file_id: Option<String>,
pub filename: Option<String>,
pub score: Option<serde_json::Number>,
pub text: Option<String>,
pub attributes: Option<Map<String, Value>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesImageGenerationCall {
pub id: Option<String>,
pub status: Option<String>,
pub result: Option<String>,
pub action: Option<String>,
pub background: Option<String>,
pub output_format: Option<String>,
pub quality: Option<String>,
pub revised_prompt: Option<String>,
pub size: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesCodeInterpreterCall {
pub id: Option<String>,
pub status: Option<String>,
pub code: Option<String>,
pub container_id: Option<String>,
pub outputs: Option<Vec<ResponsesCodeOutput>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMcpCall {
pub id: Option<String>,
pub status: Option<String>,
pub name: Option<String>,
pub server_label: Option<String>,
pub arguments: Option<String>,
pub output: Option<String>,
pub error: Option<ResponsesMcpError>,
pub approval_request_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesUrlCitation {
pub url: Option<String>,
pub title: Option<String>,
pub start_index: Option<u64>,
pub end_index: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesFileCitation {
pub file_id: Option<String>,
pub filename: Option<String>,
pub index: Option<u64>,
pub start_index: Option<u64>,
pub end_index: Option<u64>,
pub container_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesWebSearchAction {
Search {
#[serde(skip_serializing_if = "Option::is_none")]
query: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
queries: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
sources: Option<Vec<ResponsesWebSearchSource>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
OpenPage {
#[serde(skip_serializing_if = "Option::is_none")]
url: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
#[serde(alias = "find")]
FindInPage {
url: String,
pattern: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesWebSearchSource {
#[serde(rename = "type")]
pub source_type: Option<String>,
pub url: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMcpListTools {
pub id: Option<String>,
pub server_label: Option<String>,
pub tools: Option<Vec<ResponsesMcpTool>>,
pub error: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMcpTool {
pub name: Option<String>,
pub description: Option<String>,
pub input_schema: Option<crate::json_schema::JsonSchema>,
pub annotations: Option<Value>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ResponsesMcpError {
Message(String),
Detail(ResponsesMcpErrorDetail),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesMcpErrorDetail {
McpProtocolError {
code: i64,
message: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
HttpError {
code: i64,
message: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
McpToolExecutionError {
content: Value,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesToolCaller {
Direct {
#[serde(flatten)]
extra: Map<String, Value>,
},
Program {
caller_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ResponsesToolOutput {
Text(String),
Content(Vec<ResponsesInputContent>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesInputContent {
InputText {
text: String,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
InputImage {
detail: String,
#[serde(skip_serializing_if = "Option::is_none")]
file_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
image_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
InputFile {
#[serde(skip_serializing_if = "Option::is_none")]
detail: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
file_data: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
file_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
file_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
filename: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesFunctionCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub output: Option<ResponsesToolOutput>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
pub name: Option<String>,
pub namespace: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesCustomToolCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub output: Option<ResponsesToolOutput>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesSafetyCheck {
pub id: Option<String>,
pub code: Option<String>,
pub message: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesComputerCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub pending_safety_checks: Option<Vec<ResponsesSafetyCheck>>,
pub action: Option<ResponsesComputerAction>,
pub actions: Option<Vec<ResponsesComputerAction>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesComputerAction {
Click {
button: String,
x: i64,
y: i64,
#[serde(skip_serializing_if = "Option::is_none")]
keys: Option<Vec<String>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
DoubleClick {
x: i64,
y: i64,
#[serde(skip_serializing_if = "Option::is_none")]
keys: Option<Vec<String>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Drag {
path: Vec<ResponsesCoordinate>,
#[serde(skip_serializing_if = "Option::is_none")]
keys: Option<Vec<String>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Keypress {
keys: Vec<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Move {
x: i64,
y: i64,
#[serde(skip_serializing_if = "Option::is_none")]
keys: Option<Vec<String>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Screenshot {
#[serde(flatten)]
extra: Map<String, Value>,
},
Scroll {
scroll_x: i64,
scroll_y: i64,
x: i64,
y: i64,
#[serde(skip_serializing_if = "Option::is_none")]
keys: Option<Vec<String>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Type {
text: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
Wait {
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ResponsesCoordinate {
pub x: i64,
pub y: i64,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesComputerCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub output: Option<ResponsesComputerOutput>,
pub acknowledged_safety_checks: Option<Vec<ResponsesSafetyCheck>>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesComputerOutput {
ComputerScreenshot {
#[serde(skip_serializing_if = "Option::is_none")]
file_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
image_url: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesProgram {
pub id: Option<String>,
pub call_id: Option<String>,
pub code: Option<String>,
pub fingerprint: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesProgramOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub result: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesToolSearchCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub arguments: Option<Value>,
pub execution: Option<String>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesToolSearchOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub execution: Option<String>,
pub tools: Option<Vec<Map<String, Value>>>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesAdditionalTools {
pub id: Option<String>,
pub role: Option<String>,
pub tools: Option<Vec<Map<String, Value>>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesCompaction {
pub id: Option<String>,
pub encrypted_content: Option<String>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesLocalShellCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub action: Option<ResponsesLocalShellAction>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesLocalShellAction {
Exec {
command: Vec<String>,
env: BTreeMap<String, String>,
#[serde(skip_serializing_if = "Option::is_none")]
timeout_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
user: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
working_directory: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesLocalShellCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub output: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesShellCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub action: Option<ResponsesShellAction>,
pub environment: Option<ResponsesShellEnvironment>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesShellAction {
pub commands: Option<Vec<String>>,
pub max_output_length: Option<u64>,
pub timeout_ms: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesShellEnvironment {
Local {
#[serde(flatten)]
extra: Map<String, Value>,
},
ContainerReference {
container_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesShellCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub max_output_length: Option<u64>,
pub output: Option<Vec<ResponsesShellOutputChunk>>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesShellOutputChunk {
pub outcome: Option<ResponsesShellOutcome>,
pub stdout: Option<String>,
pub stderr: Option<String>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesShellOutcome {
Timeout {
#[serde(flatten)]
extra: Map<String, Value>,
},
Exit {
exit_code: i64,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesApplyPatchCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub operation: Option<ResponsesApplyPatchOperation>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesApplyPatchOperation {
CreateFile {
path: String,
diff: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
DeleteFile {
path: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
UpdateFile {
path: String,
diff: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesApplyPatchCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub output: Option<String>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMcpApprovalRequest {
pub id: Option<String>,
pub server_label: Option<String>,
pub name: Option<String>,
pub arguments: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMcpApprovalResponse {
pub id: Option<String>,
pub approval_request_id: Option<String>,
pub approve: Option<bool>,
pub reason: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

2
litellm-rust/crates/llms-types/src/formats/responses/response.rs
View file

@ -1,6 +1,6 @@
use serde_json::{Map, Value};
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ResponsesApiResponse {
pub id: String,
pub model: String,

20
litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs
View file

@ -1,3 +1,5 @@
use super::ResponsesOutputItem;
use crate::recognized::Recognized;
use serde_json::{Map, Value};
#[derive(
@ -36,7 +38,7 @@ impl ResponsesWsEventType {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ResponsesWsEvent {
#[serde(rename = "type")]
pub event_type: ResponsesWsEventType,
@ -62,7 +64,7 @@ impl ResponsesWsEvent {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq)]
pub struct ResponsesErrorFrame {
#[serde(rename = "type")]
@ -82,7 +84,7 @@ impl ResponsesErrorFrame {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq)]
pub struct ResponsesErrorBody {
#[serde(rename = "type")]
@ -90,6 +92,18 @@ pub struct ResponsesErrorBody {
pub message: String,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesEventResponse {
pub id: Option<String>,
pub model: Option<String>,
pub status: Option<String>,
pub output: Option<Vec<Recognized<ResponsesOutputItem>>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[cfg(test)]
mod tests {
use rstest::rstest;

4
litellm-rust/crates/llms-types/src/headers.rs
View file

@ -1,6 +1,6 @@
use serde_json::{Map, Value};
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ProviderSpecificHeader {
#[serde(default)]
@ -9,7 +9,7 @@ pub struct ProviderSpecificHeader {
pub extra_headers: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ProviderSpecificHeaders {
One(ProviderSpecificHeader),

51
litellm-rust/crates/llms-types/src/json_schema.rs Normal file
View file

@ -0,0 +1,51 @@
use indexmap::IndexMap;
use serde_json::{Map, Value};
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum JsonSchema {
Boolean(bool),
Object(Box<JsonSchemaObject>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum JsonSchemaType {
Name(String),
Names(Vec<String>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum JsonSchemaItems {
Schema(Box<JsonSchema>),
Tuple(Vec<JsonSchema>),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct JsonSchemaObject {
#[serde(rename = "type")]
pub schema_type: Option<JsonSchemaType>,
pub properties: Option<IndexMap<String, JsonSchema>>,
pub required: Option<Vec<String>>,
#[serde(rename = "additionalProperties")]
pub additional_properties: Option<Box<JsonSchema>>,
pub items: Option<JsonSchemaItems>,
#[serde(rename = "prefixItems")]
pub prefix_items: Option<Vec<JsonSchema>>,
#[serde(rename = "$defs")]
pub defs: Option<IndexMap<String, JsonSchema>>,
#[serde(rename = "$ref")]
pub reference: Option<String>,
#[serde(rename = "anyOf")]
pub any_of: Option<Vec<JsonSchema>>,
#[serde(rename = "allOf")]
pub all_of: Option<Vec<JsonSchema>>,
#[serde(rename = "oneOf")]
pub one_of: Option<Vec<JsonSchema>>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

1
litellm-rust/crates/llms-types/src/lib.rs
View file

@ -6,6 +6,7 @@ macro_rules_attribute::attribute_alias! {
pub mod formats;
pub mod headers;
pub mod json_schema;
pub mod providers;
pub mod recognized;
pub mod serde_compat;

18
litellm-rust/crates/llms-types/src/providers/AGENTS.md Normal file
View file

@ -0,0 +1,18 @@
# references
These links describe the provider-specific wire types in this directory. Shared Messages payload references live in `../formats/messages/AGENTS.md`
## anthropic.rs
`AnthropicBeta`, `BetaSet` and `BetaProvider` represent the `anthropic-beta` header values, their
wire spelling and per-host support
- https://platform.claude.com/docs/en/api/beta-headers.md
## minimax.rs
MiniMax's Messages reference documents its image, video, and mid-conversation system content extensions. Its cache reference documents `cache_control` on those blocks
- https://platform.minimax.io/docs/api-reference/text-chat-anthropic
- https://platform.minimax.io/docs/api-reference/text-chat-anthropic.md
- https://platform.minimax.io/docs/api-reference/anthropic-api-compatible-cache.md

368
litellm-rust/crates/llms-types/src/providers/anthropic.rs
View file

@ -7,6 +7,34 @@ use std::{
str::FromStr,
};
/// A provider column of `litellm/anthropic_beta_headers_config.json`: which betas a host accepts
/// and under which name.
#[derive(
Clone, Copy, Debug, PartialEq, Eq, Hash, strum::AsRefStr, strum::Display, strum::EnumString,
)]
#[strum(serialize_all = "snake_case")]
pub enum BetaProvider {
Anthropic,
AzureAi,
BedrockConverse,
Bedrock,
BedrockMantle,
VertexAi,
Databricks,
}
impl BetaProvider {
pub const ALL: &[Self] = &[
Self::Anthropic,
Self::AzureAi,
Self::BedrockConverse,
Self::Bedrock,
Self::BedrockMantle,
Self::VertexAi,
Self::Databricks,
];
}
/// One value of the `anthropic-beta` header. Equality, ordering and hashing follow the wire
/// string, so a value parsed from a caller's header never disagrees with the matching variant.
#[derive(Clone, Debug, strum::AsRefStr, strum::Display, strum::EnumString)]
@ -35,12 +63,66 @@ pub enum AnthropicBeta {
PerTurnControl20260701,
#[strum(serialize = "dangerous-tool-use-2026-09-03")]
DangerousToolUse20260903,
#[strum(serialize = "bash_20241022")]
Bash20241022,
#[strum(serialize = "bash_20250124")]
Bash20250124,
#[strum(serialize = "claude-code-20250219")]
ClaudeCode20250219,
#[strum(serialize = "code-execution-2025-08-25")]
CodeExecution20250825,
#[strum(serialize = "computer-use-2025-01-24")]
ComputerUse20250124,
#[strum(serialize = "computer-use-2025-11-24")]
ComputerUse20251124,
#[strum(serialize = "context-1m-2025-08-07")]
Context1m20250807,
#[strum(serialize = "effort-2025-11-24")]
Effort20251124,
#[strum(serialize = "files-api-2025-04-14")]
FilesApi20250414,
#[strum(serialize = "fine-grained-tool-streaming-2025-05-14")]
FineGrainedToolStreaming20250514,
#[strum(serialize = "interleaved-thinking-2025-05-14")]
InterleavedThinking20250514,
#[strum(serialize = "mcp-client-2025-04-04")]
McpClient20250404,
#[strum(serialize = "mcp-client-2025-11-20")]
McpClient20251120,
#[strum(serialize = "mcp-servers-2025-12-04")]
McpServers20251204,
#[strum(serialize = "mid-conversation-output-config-2026-07-01")]
MidConversationOutputConfig20260701,
#[strum(serialize = "mid-conversation-tool-changes-2026-07-01")]
MidConversationToolChanges20260701,
#[strum(serialize = "output-128k-2025-02-19")]
Output128k20250219,
#[strum(serialize = "prompt-caching-scope-2026-01-05")]
PromptCachingScope20260105,
#[strum(serialize = "skills-2025-10-02")]
Skills20251002,
#[strum(serialize = "structured-output-2024-03-01")]
StructuredOutput20240301,
#[strum(serialize = "text_editor_20241022")]
TextEditor20241022,
#[strum(serialize = "text_editor_20250124")]
TextEditor20250124,
#[strum(serialize = "thinking-binding-controls-2026-08-01")]
ThinkingBindingControls20260801,
#[strum(serialize = "thinking-display-updates-2026-08-18")]
ThinkingDisplayUpdates20260818,
#[strum(serialize = "token-efficient-tools-2025-02-19")]
TokenEfficientTools20250219,
#[strum(serialize = "tool-examples-2025-10-29")]
ToolExamples20251029,
#[strum(serialize = "tool-search-tool-2025-10-19")]
ToolSearchTool20251019,
#[strum(default, transparent)]
Other(String),
}
impl AnthropicBeta {
pub const KNOWN: [Self; 12] = [
pub const KNOWN: [Self; 39] = [
Self::Oauth20250420,
Self::WebFetch20250910,
Self::WebSearch20250305,
@ -53,11 +135,145 @@ impl AnthropicBeta {
Self::AdvisorTool20260301,
Self::PerTurnControl20260701,
Self::DangerousToolUse20260903,
Self::Bash20241022,
Self::Bash20250124,
Self::ClaudeCode20250219,
Self::CodeExecution20250825,
Self::ComputerUse20250124,
Self::ComputerUse20251124,
Self::Context1m20250807,
Self::Effort20251124,
Self::FilesApi20250414,
Self::FineGrainedToolStreaming20250514,
Self::InterleavedThinking20250514,
Self::McpClient20250404,
Self::McpClient20251120,
Self::McpServers20251204,
Self::MidConversationOutputConfig20260701,
Self::MidConversationToolChanges20260701,
Self::Output128k20250219,
Self::PromptCachingScope20260105,
Self::Skills20251002,
Self::StructuredOutput20240301,
Self::TextEditor20241022,
Self::TextEditor20250124,
Self::ThinkingBindingControls20260801,
Self::ThinkingDisplayUpdates20260818,
Self::TokenEfficientTools20250219,
Self::ToolExamples20251029,
Self::ToolSearchTool20251019,
];
pub fn as_str(&self) -> &str {
self.as_ref()
}
pub fn on(&self, provider: BetaProvider) -> Option<Self> {
let resolved = match self {
Self::Other(raw) => raw.parse().unwrap(),
_ => self.clone(),
};
if resolved.rejected_by().contains(&provider) {
return None;
}
Some(match (resolved, provider) {
(
Self::AdvancedToolUse20251120,
BetaProvider::Bedrock | BetaProvider::BedrockMantle | BetaProvider::VertexAi,
) => Self::ToolSearchTool20251019,
(beta, _) => beta,
})
}
fn rejected_by(&self) -> &'static [BetaProvider] {
match self {
Self::AdvancedToolUse20251120 | Self::ContextManagement20250627 => {
&[BetaProvider::BedrockConverse]
}
Self::AdvisorTool20260301 | Self::Compact20260904 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Bash20241022
| Self::Bash20250124
| Self::McpServers20251204
| Self::StructuredOutput20240301
| Self::TextEditor20241022
| Self::TextEditor20250124 => BetaProvider::ALL,
Self::ClaudeCode20250219 | Self::ToolExamples20251029 => &[
BetaProvider::Anthropic,
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::CodeExecution20250825
| Self::FilesApi20250414
| Self::McpClient20250404
| Self::McpClient20251120
| Self::PromptCachingScope20260105
| Self::Skills20251002
| Self::WebFetch20250910 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
Self::DangerousToolUse20260903 => {
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
}
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
],
Self::FineGrainedToolStreaming20250514 => {
&[BetaProvider::AzureAi, BetaProvider::VertexAi]
}
Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
&[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
}
Self::MidConversationOutputConfig20260701
| Self::MidConversationToolChanges20260701 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::VertexAi,
],
Self::PerTurnControl20260701 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Databricks,
],
Self::Other(_) => BetaProvider::ALL,
}
}
}
impl PartialEq for AnthropicBeta {
@ -149,10 +365,20 @@ impl fmt::Display for BetaSet {
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use indexmap::IndexMap;
use rstest::rstest;
use super::*;
fn beta_headers_config() -> IndexMap<String, serde_json::Value> {
serde_json::from_str(include_str!(
"../../../../../litellm/anthropic_beta_headers_config.json"
))
.unwrap()
}
fn set(header: &str) -> BetaSet {
header.parse().unwrap_or_else(|never| match never {})
}
@ -171,7 +397,34 @@ mod tests {
AnthropicBeta::FastMode20260201,
AnthropicBeta::AdvisorTool20260301,
AnthropicBeta::PerTurnControl20260701,
AnthropicBeta::DangerousToolUse20260903
AnthropicBeta::DangerousToolUse20260903,
AnthropicBeta::Bash20241022,
AnthropicBeta::Bash20250124,
AnthropicBeta::ClaudeCode20250219,
AnthropicBeta::CodeExecution20250825,
AnthropicBeta::ComputerUse20250124,
AnthropicBeta::ComputerUse20251124,
AnthropicBeta::Context1m20250807,
AnthropicBeta::Effort20251124,
AnthropicBeta::FilesApi20250414,
AnthropicBeta::FineGrainedToolStreaming20250514,
AnthropicBeta::InterleavedThinking20250514,
AnthropicBeta::McpClient20250404,
AnthropicBeta::McpClient20251120,
AnthropicBeta::McpServers20251204,
AnthropicBeta::MidConversationOutputConfig20260701,
AnthropicBeta::MidConversationToolChanges20260701,
AnthropicBeta::Output128k20250219,
AnthropicBeta::PromptCachingScope20260105,
AnthropicBeta::Skills20251002,
AnthropicBeta::StructuredOutput20240301,
AnthropicBeta::TextEditor20241022,
AnthropicBeta::TextEditor20250124,
AnthropicBeta::ThinkingBindingControls20260801,
AnthropicBeta::ThinkingDisplayUpdates20260818,
AnthropicBeta::TokenEfficientTools20250219,
AnthropicBeta::ToolExamples20251029,
AnthropicBeta::ToolSearchTool20251019
)]
beta: AnthropicBeta,
) {
@ -181,6 +434,117 @@ mod tests {
assert!(AnthropicBeta::KNOWN.contains(&beta));
}
#[rstest]
fn known_betas_are_exactly_the_config_keys() {
let config = beta_headers_config();
let config_keys: BTreeSet<String> = config
.iter()
.filter(|(provider, _)| provider.as_str() != "description")
.flat_map(|(_, column)| {
column
.as_object()
.into_iter()
.flat_map(|values| values.keys().cloned())
})
.collect();
let known_keys: BTreeSet<String> = AnthropicBeta::KNOWN
.iter()
.map(|beta| beta.as_str().to_string())
.collect();
assert_eq!(known_keys, config_keys);
}
#[rstest]
#[case::anthropic(BetaProvider::Anthropic)]
#[case::azure_ai(BetaProvider::AzureAi)]
#[case::bedrock_converse(BetaProvider::BedrockConverse)]
#[case::bedrock(BetaProvider::Bedrock)]
#[case::bedrock_mantle(BetaProvider::BedrockMantle)]
#[case::vertex_ai(BetaProvider::VertexAi)]
#[case::databricks(BetaProvider::Databricks)]
fn provider_columns_match_beta_provider_all(#[case] provider: BetaProvider) {
let config = beta_headers_config();
let config_columns: Vec<String> = config
.keys()
.filter(|column| column.as_str() != "description")
.cloned()
.collect();
let beta_provider_columns: Vec<String> =
BetaProvider::ALL.iter().map(ToString::to_string).collect();
assert_eq!(config_columns, beta_provider_columns);
assert_eq!(
provider.to_string().parse::<BetaProvider>().unwrap(),
provider
);
}
#[rstest]
fn on_matches_every_config_cell() {
let config = beta_headers_config();
for provider in BetaProvider::ALL {
for beta in &AnthropicBeta::KNOWN {
let expected = config
.get(provider.as_ref())
.and_then(serde_json::Value::as_object)
.and_then(|column| column.get(beta.as_str()))
.and_then(serde_json::Value::as_str)
.map(str::to_string);
let actual = beta.on(*provider).map(|name| name.to_string());
assert_eq!(actual, expected, "provider {provider}, beta {beta}");
if matches!(beta, AnthropicBeta::AdvancedToolUse20251120)
&& matches!(
provider,
BetaProvider::Bedrock
| BetaProvider::BedrockMantle
| BetaProvider::VertexAi
)
{
let parsed: AnthropicBeta = actual.as_deref().unwrap().parse().unwrap();
assert!(AnthropicBeta::KNOWN.contains(&parsed));
assert!(!matches!(parsed, AnthropicBeta::Other(_)));
}
}
}
}
#[rstest]
#[case::renamed(
"advanced-tool-use-2025-11-20",
BetaProvider::Bedrock,
Some(AnthropicBeta::ToolSearchTool20251019)
)]
#[case::kept(
"oauth-2025-04-20",
BetaProvider::Anthropic,
Some(AnthropicBeta::Oauth20250420)
)]
#[case::rejected("effort-2025-11-24", BetaProvider::VertexAi, None)]
#[case::unknown("example-beta-2099-01-01", BetaProvider::Anthropic, None)]
fn on_resolves_known_spellings_held_as_other(
#[case] raw: &str,
#[case] provider: BetaProvider,
#[case] expected: Option<AnthropicBeta>,
) {
let actual = AnthropicBeta::Other(raw.to_string()).on(provider);
assert_eq!(actual, expected);
assert!(!matches!(actual, Some(AnthropicBeta::Other(_))));
}
#[rstest]
#[case::anthropic(BetaProvider::Anthropic)]
#[case::azure_ai(BetaProvider::AzureAi)]
#[case::bedrock_converse(BetaProvider::BedrockConverse)]
#[case::bedrock(BetaProvider::Bedrock)]
#[case::bedrock_mantle(BetaProvider::BedrockMantle)]
#[case::vertex_ai(BetaProvider::VertexAi)]
#[case::databricks(BetaProvider::Databricks)]
fn unknown_beta_is_rejected_by_every_provider(#[case] provider: BetaProvider) {
assert_eq!(
AnthropicBeta::Other("example-beta-2099-01-01".into()).on(provider),
None
);
}
#[test]
fn unknown_values_are_kept_verbatim() {
let parsed: AnthropicBeta = "claude-code-20250219".parse().unwrap();

59
litellm-rust/crates/llms-types/src/providers/minimax.rs Normal file
View file

@ -0,0 +1,59 @@
use serde_json::{Map, Value};
use crate::formats::messages::CacheControl;
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MinimaxMessagesContentBlock {
Image(MinimaxMediaBlock),
Video(MinimaxMediaBlock),
MidConvSystem {
text: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MinimaxMediaBlock {
pub source: MinimaxMediaSource,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MinimaxMediaSource {
Base64 {
media_type: String,
data: String,
#[serde(flatten)]
options: MinimaxMediaOptions,
},
Url {
url: String,
#[serde(flatten)]
options: MinimaxMediaOptions,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MinimaxMediaOptions {
pub detail: Option<MinimaxMediaDetail>,
pub fps: Option<serde_json::Number>,
pub max_long_side_pixel: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum MinimaxMediaDetail {
Low,
Default,
High,
}

1
litellm-rust/crates/llms-types/src/providers/mod.rs
View file

@ -1 +1,2 @@
pub mod anthropic;
pub mod minimax;

2
litellm-rust/crates/llms-types/src/recognized.rs
View file

@ -1,6 +1,6 @@
use serde_json::Value;
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum Recognized<T> {
Known(T),

163
litellm-rust/crates/llms-types/tests/chat.rs Normal file
View file

@ -0,0 +1,163 @@
use litellm_llms_types::formats::chat_completions::{
ChatContentPart, ChatLogprobs, ChatMediaUrl, PromptCacheMode,
};
use litellm_llms_types::formats::messages::ContentSource;
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::parts(json!([
{"type":"text","text":"hello","cache_control":{"type":"ephemeral"},"prompt_cache_breakpoint":{"mode":"explicit"}},
{"type":"image_url","image_url":{"url":"https://example.test/image","detail":"high","format":"image/png"}},
{"type":"video_url","video_url":"https://example.test/video"},
{"type":"input_audio","input_audio":{"data":"AA==","format":"wav"}},
{"type":"file","file":{"file_id":"file_1","file_data":"JVBE","filename":"a.mp4","format":"video/mp4","detail":"low","video_metadata":{"fps":1,"start_offset":"1s","end_offset":"2s"}}},
{"type":"document","source":{"type":"text","media_type":"text/plain","data":"doc"},"title":"T","context":"C","citations":{"enabled":true}},
{"type":"refusal","refusal":"refused"}
]))]
fn content_parts_round_trip(#[case] wire: Value) {
let parts: Vec<ChatContentPart> = serde_json::from_value(wire.clone()).unwrap();
let [
ChatContentPart::Text {
text,
cache_control,
prompt_cache_breakpoint: Some(breakpoint),
extra: text_extra,
},
ChatContentPart::ImageUrl {
image_url: ChatMediaUrl::Parameters(image),
prompt_cache_breakpoint: None,
..
},
ChatContentPart::VideoUrl {
video_url: ChatMediaUrl::Url(video),
..
},
ChatContentPart::InputAudio { input_audio, .. },
ChatContentPart::File { file, .. },
ChatContentPart::Document {
source,
title: Some(title),
context: Some(context),
citations: Some(citations),
..
},
ChatContentPart::Refusal { refusal, .. },
] = parts.as_slice()
else {
panic!("expected typed content parts");
};
assert_eq!(text, "hello");
assert_eq!(
cache_control.as_ref().unwrap().cache_type.as_deref(),
Some("ephemeral")
);
assert_eq!(breakpoint.mode, PromptCacheMode::Explicit);
assert!(breakpoint.extra.is_empty());
assert!(text_extra.is_empty());
assert_eq!(image.url, "https://example.test/image");
assert_eq!(image.detail.as_deref(), Some("high"));
assert_eq!(image.format.as_deref(), Some("image/png"));
assert!(image.extra.is_empty());
assert_eq!(video, "https://example.test/video");
assert_eq!(input_audio.data, "AA==");
assert_eq!(input_audio.format, "wav");
assert_eq!(file.file_id.as_deref(), Some("file_1"));
assert_eq!(file.file_data.as_deref(), Some("JVBE"));
assert_eq!(file.filename.as_deref(), Some("a.mp4"));
assert_eq!(file.format.as_deref(), Some("video/mp4"));
assert_eq!(file.detail.as_deref(), Some("low"));
assert!(file.extra.is_empty());
let metadata = file.video_metadata.as_ref().unwrap();
assert_eq!(metadata.fps, Some(1.into()));
assert_eq!(metadata.start_offset.as_deref(), Some("1s"));
assert_eq!(metadata.end_offset.as_deref(), Some("2s"));
assert!(metadata.extra.is_empty());
let ContentSource::Text {
data, media_type, ..
} = source.as_ref()
else {
panic!("expected document text source");
};
assert_eq!(data, "doc");
assert_eq!(media_type, "text/plain");
assert_eq!(title, "T");
assert_eq!(context, "C");
assert_eq!(citations.enabled, Some(true));
assert_eq!(refusal, "refused");
assert_eq!(serde_json::to_value(parts).unwrap(), wire);
}
#[rstest]
fn logprobs_round_trip_with_tokens_and_alternatives() {
let wire = json!({
"content":[{"token":"hi","logprob":-1,"bytes":[104,105],"top_logprobs":[{"token":"hey","logprob":-2.5,"bytes":[104]}]}],
"refusal":[{"token":"refused","logprob":-3}]
});
let logprobs: ChatLogprobs = serde_json::from_value(wire.clone()).unwrap();
let token = &logprobs.content.as_ref().unwrap()[0];
assert_eq!(token.token, "hi");
assert_eq!(token.logprob, (-1).into());
assert_eq!(token.bytes.as_deref(), Some([104, 105].as_slice()));
let alternative = &token.top_logprobs.as_ref().unwrap()[0];
assert_eq!(alternative.token, "hey");
assert_eq!(alternative.bytes.as_deref(), Some([104].as_slice()));
assert_eq!(logprobs.refusal.as_ref().unwrap()[0].token, "refused");
assert!(logprobs.refusal.as_ref().unwrap()[0].top_logprobs.is_none());
assert_eq!(serde_json::to_value(logprobs).unwrap(), wire);
}
#[rstest]
#[case::text(json!({"type":"text","text":false}))]
#[case::missing_image(json!({"type":"image_url"}))]
#[case::audio_shape(json!({"type":"input_audio","input_audio":{"data":7,"format":"wav"}}))]
#[case::audio_missing_format(json!({"type":"input_audio","input_audio":{"data":"AA=="}}))]
#[case::image_missing_url(json!({"type":"image_url","image_url":{"detail":"high"}}))]
#[case::file_metadata(json!({"type":"file","file":{"video_metadata":{"fps":"fast"}}}))]
#[case::document_source(json!({"type":"document","source":{"type":"url","url":false}}))]
#[case::refusal_missing_text(json!({"type":"refusal"}))]
#[case::file_missing_file(json!({"type":"file"}))]
#[case::document_missing_source(json!({"type":"document"}))]
#[case::video_missing_url(json!({"type":"video_url"}))]
#[case::cache_control_shape(json!({"type":"text","text":"t","cache_control":"ephemeral"}))]
#[case::breakpoint_missing_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{}}))]
#[case::breakpoint_unknown_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{"mode":"auto"}}))]
#[case::unknown_tag(json!({"type":"future"}))]
fn content_parts_reject_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ChatContentPart>(wire).is_err());
}
#[rstest]
fn partial_file_preserves_extensions_and_omits_null_optionals() {
let wire = json!({"type":"file","file":{"file_id":null,"filename":"a.pdf","extension":[1,null]},"future":true});
let part: ChatContentPart = serde_json::from_value(wire).unwrap();
let ChatContentPart::File {
file,
prompt_cache_breakpoint: None,
extra,
} = &part
else {
panic!("expected file")
};
assert!(file.file_id.is_none());
assert!(file.file_data.is_none());
assert_eq!(file.filename.as_deref(), Some("a.pdf"));
assert_eq!(extra["future"], json!(true));
assert_eq!(
serde_json::to_value(part).unwrap(),
json!({"type":"file","file":{"filename":"a.pdf","extension":[1,null]},"future":true})
);
}
#[rstest]
#[case::missing_token(json!({"logprob":-1}))]
#[case::missing_logprob(json!({"token":"hi"}))]
#[case::wrong_bytes(json!({"token":"hi","logprob":-1,"bytes":[256]}))]
fn token_logprobs_reject_malformed_fields(#[case] wire: Value) {
assert!(
serde_json::from_value::<litellm_llms_types::formats::chat_completions::ChatTokenLogprob>(
wire
)
.is_err()
);
}

148
litellm-rust/crates/llms-types/tests/json_schema.rs Normal file
View file

@ -0,0 +1,148 @@
use litellm_llms_types::json_schema::{
JsonSchema, JsonSchemaItems, JsonSchemaObject, JsonSchemaType,
};
use rstest::rstest;
use serde::{Serialize, de::DeserializeOwned};
use serde_json::{Value, json};
fn round_trip<T>(wire: Value) -> T
where
T: DeserializeOwned + Serialize,
{
let parsed: T = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(&parsed).unwrap(), wire);
parsed
}
#[rstest]
#[case::boolean(json!(false))]
#[case::nested(json!({
"type":["object","null"],
"properties":{"nested":{"type":"array","items":{"$ref":"#/$defs/item"}},"free":true},
"additionalProperties":{"type":"string"},
"$defs":{"item":{"anyOf":[{"type":"string"},false]}},
"required":["nested"],
"enum":[{"custom":[1,null]}],
"future":null
}))]
fn recursive_schema_round_trips(#[case] wire: Value) {
let schema = round_trip::<JsonSchema>(wire);
match schema {
JsonSchema::Boolean(allowed) => assert!(!allowed),
JsonSchema::Object(schema) => {
assert_eq!(
schema.schema_type,
Some(JsonSchemaType::Names(vec!["object".into(), "null".into()]))
);
let properties = schema.properties.as_ref().unwrap();
let JsonSchema::Object(nested) = &properties["nested"] else {
panic!("expected nested schema")
};
let Some(JsonSchemaItems::Schema(items)) = &nested.items else {
panic!("expected single items schema")
};
let JsonSchema::Object(items) = items.as_ref() else {
panic!("expected items schema")
};
assert_eq!(items.reference.as_deref(), Some("#/$defs/item"));
assert_eq!(properties["free"], JsonSchema::Boolean(true));
let JsonSchema::Object(definition) = &schema.defs.as_ref().unwrap()["item"] else {
panic!("expected schema definition")
};
assert_eq!(
definition.any_of.as_ref().unwrap()[1],
JsonSchema::Boolean(false)
);
assert_eq!(schema.extra["enum"], json!([{"custom":[1,null]}]));
}
}
}
#[rstest]
fn schema_maps_keep_wire_key_order() {
let wire = r#"{"type":"object","properties":{"zeta":{"type":"string"},"alpha":{"type":"integer"}},"$defs":{"y":true,"b":false}}"#;
let schema: JsonSchemaObject = serde_json::from_str(wire).unwrap();
let names: Vec<&str> = schema
.properties
.as_ref()
.unwrap()
.keys()
.map(String::as_str)
.collect();
assert_eq!(names, ["zeta", "alpha"]);
assert_eq!(serde_json::to_string(&schema).unwrap(), wire);
}
#[rstest]
fn schema_object_round_trips_supported_keywords() {
let schema = round_trip::<JsonSchemaObject>(json!({
"type":"object",
"properties":{"name":{"type":"string"}},
"required":["name"],
"additionalProperties":false,
"$ref":"#/$defs/value",
"strict":true,
"extension":{"nested":[1,null]}
}));
assert_eq!(
schema.schema_type,
Some(JsonSchemaType::Name("object".into()))
);
assert_eq!(schema.required.as_deref(), Some(["name".into()].as_slice()));
assert_eq!(
schema.additional_properties.as_deref(),
Some(&JsonSchema::Boolean(false))
);
assert_eq!(schema.reference.as_deref(), Some("#/$defs/value"));
assert_eq!(schema.strict, Some(true));
}
#[rstest]
#[case::scalar(json!(7))]
#[case::properties_shape(json!({"properties":[]}))]
#[case::nested_schema(json!({"properties":{"name":7}}))]
#[case::schema_type(json!({"type":["string",7]}))]
#[case::items_shape(json!({"items":7}))]
#[case::tuple_member(json!({"items":[{"type":"string"},7]}))]
#[case::prefix_items_shape(json!({"prefixItems":{"type":"string"}}))]
#[case::composite_shape(json!({"anyOf":["string"]}))]
fn schemas_reject_malformed_known_keywords(#[case] wire: Value) {
assert!(serde_json::from_value::<JsonSchema>(wire).is_err());
}
#[rstest]
#[case::draft_07_tuple(json!({"type":"array","items":[{"type":"string"},true]}))]
#[case::draft_2020_12_tuple(json!({"type":"array","prefixItems":[{"type":"string"},true],"items":false}))]
fn tuple_schemas_expose_positional_members(#[case] wire: Value) {
let schema = round_trip::<JsonSchemaObject>(wire);
let members = match (&schema.items, &schema.prefix_items) {
(Some(JsonSchemaItems::Tuple(members)), None) => members,
(Some(JsonSchemaItems::Schema(rest)), Some(members)) => {
assert_eq!(rest.as_ref(), &JsonSchema::Boolean(false));
members
}
_ => panic!("expected positional members"),
};
let [JsonSchema::Object(first), JsonSchema::Boolean(true)] = members.as_slice() else {
panic!("expected string schema then true");
};
assert_eq!(
first.schema_type,
Some(JsonSchemaType::Name("string".into()))
);
assert!(schema.extra.is_empty());
}
#[rstest]
fn empty_schema_omits_null_optionals_and_preserves_arbitrary_keywords() {
let schema: JsonSchemaObject = serde_json::from_value(
json!({"type":null,"properties":null,"const":{"arbitrary":[1,null]},"future":null}),
)
.unwrap();
assert!(schema.schema_type.is_none());
assert!(schema.properties.is_none());
assert_eq!(
serde_json::to_value(schema).unwrap(),
json!({"const":{"arbitrary":[1,null]},"future":null})
);
}

9
litellm-rust/crates/llms-types/tests/messages_request.rs
View file

@ -33,6 +33,15 @@ fn content_block_type_schema_remains_a_string() {
#[case::compaction("compaction", ContentBlockType::Compaction)]
#[case::advisor_result("advisor_tool_result", ContentBlockType::AdvisorToolResult)]
#[case::web_search_result("web_search_tool_result", ContentBlockType::WebSearchToolResult)]
#[case::image("image", ContentBlockType::Other("image".into()))]
#[case::document("document", ContentBlockType::Other("document".into()))]
#[case::tool_addition("tool_addition", ContentBlockType::Other("tool_addition".into()))]
#[case::tool_removal("tool_removal", ContentBlockType::Other("tool_removal".into()))]
#[case::advisor("advisor_result", ContentBlockType::Other("advisor_result".into()))]
#[case::web_search_error(
"web_search_tool_result_error",
ContentBlockType::Other("web_search_tool_result_error".into())
)]
#[case::future_block("future_block", ContentBlockType::Other("future_block".into()))]
#[case::case_sensitive("Tool_Use", ContentBlockType::Other("Tool_Use".into()))]
#[case::empty("", ContentBlockType::Other(String::new()))]

1512
litellm-rust/crates/llms-types/tests/messages_types.rs Normal file
View file

File diff suppressed because it is too large Load diff

82
litellm-rust/crates/llms-types/tests/minimax.rs Normal file
View file

@ -0,0 +1,82 @@
use litellm_llms_types::providers::minimax::{
MinimaxMediaDetail, MinimaxMediaSource, MinimaxMessagesContentBlock,
};
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::image(json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA==","detail":"low"}}))]
#[case::video(json!({"type":"video","source":{"type":"url","url":"https://example.test/video","detail":"high","fps":1,"max_long_side_pixel":1024,"future":null},"cache_control":{"type":"ephemeral"}}))]
#[case::mid_conversation_system(json!({"type":"mid_conv_system","text":"instruction","future":null}))]
fn provider_content_blocks_round_trip(#[case] wire: Value) {
let block: MinimaxMessagesContentBlock = serde_json::from_value(wire.clone()).unwrap();
match &block {
MinimaxMessagesContentBlock::Image(image) => {
let MinimaxMediaSource::Base64 {
media_type,
data,
options,
} = &image.source
else {
panic!("expected base64 image source");
};
assert_eq!((media_type.as_str(), data.as_str()), ("image/png", "AA=="));
assert_eq!(options.detail, Some(MinimaxMediaDetail::Low));
assert!(options.extra.is_empty());
}
MinimaxMessagesContentBlock::Video(video) => {
let MinimaxMediaSource::Url { url, options } = &video.source else {
panic!("expected URL video source");
};
assert_eq!(url, "https://example.test/video");
assert_eq!(options.detail, Some(MinimaxMediaDetail::High));
assert_eq!(options.fps, Some(1.into()));
assert_eq!(options.max_long_side_pixel, Some(1024));
assert_eq!(options.extra["future"], Value::Null);
assert_eq!(
video.cache_control.as_ref().unwrap().cache_type.as_deref(),
Some("ephemeral")
);
}
MinimaxMessagesContentBlock::MidConvSystem { text, extra } => {
assert_eq!(text, "instruction");
assert_eq!(extra["future"], Value::Null);
}
}
assert_eq!(serde_json::to_value(block).unwrap(), wire);
}
#[rstest]
#[case::missing_source(json!({"type":"video"}))]
#[case::bad_source_tag(json!({"type":"image","source":{"type":"future"}}))]
#[case::url_without_url(json!({"type":"video","source":{"type":"url"}}))]
#[case::base64_without_data(json!({"type":"image","source":{"type":"base64","media_type":"image/png"}}))]
#[case::bad_detail(json!({"type":"image","source":{"type":"url","url":"u","detail":7}}))]
#[case::bad_fps(json!({"type":"video","source":{"type":"url","url":"u","fps":"fast"}}))]
#[case::missing_text(json!({"type":"mid_conv_system"}))]
fn provider_content_rejects_malformed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<MinimaxMessagesContentBlock>(wire).is_err());
}
#[rstest]
fn partial_media_options_omit_null_optionals() {
let block: MinimaxMessagesContentBlock = serde_json::from_value(json!({
"type":"video",
"source":{"type":"url","url":"u","detail":null,"fps":null,"future":null},
"cache_control":null
}))
.unwrap();
let MinimaxMessagesContentBlock::Video(video) = &block else {
panic!("expected video")
};
let MinimaxMediaSource::Url { options, .. } = &video.source else {
panic!("expected URL source");
};
assert!(options.detail.is_none());
assert!(options.fps.is_none());
assert!(video.cache_control.is_none());
assert_eq!(
serde_json::to_value(block).unwrap(),
json!({"type":"video","source":{"type":"url","url":"u","future":null}})
);
}

38
litellm-rust/crates/llms-types/tests/ocr.rs
View file

@ -1,4 +1,4 @@
use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrPage};
use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrBoundingBox, OcrDocument, OcrPage};
use rstest::rstest;
use serde_json::{Map, Value, json};
@ -102,3 +102,39 @@ fn response_serialization_preserves_extensions_and_native_presence(
assert_eq!(decoded.provider_native_response, native);
assert_eq!(decoded.into_json(), serialized);
}
#[rstest]
fn bounding_box_exposes_corner_coordinates_and_keeps_extensions() {
let wire = json!({"top_left_x":1,"top_left_y":2.5,"bottom_right_x":30,"bottom_right_y":40,"future":true});
let bounds: OcrBoundingBox = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(bounds.top_left_x, Some(1.into()));
assert_eq!(
bounds
.top_left_y
.as_ref()
.and_then(serde_json::Number::as_f64),
Some(2.5)
);
assert_eq!(bounds.bottom_right_x, Some(30.into()));
assert_eq!(bounds.bottom_right_y, Some(40.into()));
assert_eq!(Value::Object(bounds.extra.clone()), json!({"future":true}));
assert_eq!(serde_json::to_value(bounds).unwrap(), wire);
}
#[rstest]
fn partial_bounding_box_omits_null_corners() {
let bounds: OcrBoundingBox =
serde_json::from_value(json!({"top_left_x":null,"bottom_right_y":4})).unwrap();
assert!(bounds.top_left_x.is_none());
assert_eq!(
serde_json::to_value(bounds).unwrap(),
json!({"bottom_right_y":4})
);
}
#[rstest]
#[case::string_corner(json!({"top_left_x":"1"}))]
#[case::array_corner(json!({"bottom_right_y":[4]}))]
fn bounding_box_rejects_non_numeric_corners(#[case] wire: Value) {
assert!(serde_json::from_value::<OcrBoundingBox>(wire).is_err());
}

671
litellm-rust/crates/llms-types/tests/responses.rs
View file

@ -1,5 +1,31 @@
use litellm_llms_types::formats::responses::streaming_websocket::ResponsesWsEventType;
use litellm_llms_types::formats::chat_completions::{PromptCacheBreakpoint, PromptCacheMode};
use litellm_llms_types::formats::responses::{
ResponsesAdditionalTools, ResponsesApplyPatchCall, ResponsesApplyPatchCallOutput,
ResponsesApplyPatchOperation, ResponsesCodeOutput, ResponsesCompaction,
ResponsesComputerAction, ResponsesComputerCall, ResponsesComputerCallOutput,
ResponsesComputerOutput, ResponsesContentPart, ResponsesCoordinate, ResponsesCustomToolCall,
ResponsesCustomToolCallOutput, ResponsesFunctionCall, ResponsesFunctionCallOutput,
ResponsesImageGenerationCall, ResponsesInputContent, ResponsesLocalShellAction,
ResponsesLocalShellCall, ResponsesLocalShellCallOutput, ResponsesMcpApprovalRequest,
ResponsesMcpApprovalResponse, ResponsesMcpError, ResponsesMcpErrorDetail, ResponsesOutputItem,
ResponsesProgram, ResponsesProgramOutput, ResponsesSafetyCheck, ResponsesShellAction,
ResponsesShellCall, ResponsesShellCallOutput, ResponsesShellEnvironment, ResponsesShellOutcome,
ResponsesShellOutputChunk, ResponsesToolCaller, ResponsesToolOutput, ResponsesToolSearchCall,
ResponsesToolSearchOutput, ResponsesWebSearchAction,
streaming_websocket::{ResponsesEventResponse, ResponsesWsEventType},
};
use litellm_llms_types::recognized::Recognized;
use rstest::rstest;
use serde::{Serialize, de::DeserializeOwned};
use serde_json::{Value, json};
fn round_trip<T>(wire: Value)
where
T: DeserializeOwned + Serialize,
{
let parsed: T = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(parsed).unwrap(), wire);
}
#[rstest]
#[case::create("response.create", ResponsesWsEventType::ResponseCreate)]
@ -46,3 +72,646 @@ fn websocket_event_type_schema_is_open_string() {
Some(&serde_json::json!("string"))
);
}
#[rstest]
#[case::message(json!({"type":"message","role":"assistant","content":[{"type":"output_text","text":"answer","annotations":[{"type":"url_citation","url":"https://example.test","start_index":0,"end_index":6}]}]}))]
#[case::function_call(json!({"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{\"query\":\"q\"}"}))]
#[case::custom_tool(json!({"type":"custom_tool_call","name":"lookup","input":"q"}))]
#[case::reasoning(json!({"type":"reasoning","summary":[{"type":"summary_text","text":"summary"}],"encrypted_content":"opaque"}))]
#[case::web_search(json!({"type":"web_search_call","action":{"type":"search","queries":["q"],"sources":[{"type":"url","url":"https://example.test"}]}}))]
#[case::file_search(json!({"type":"file_search_call","queries":["q"],"results":[{"file_id":"file_1","score":1,"attributes":{"custom":[1,null]}}]}))]
#[case::code(json!({"type":"code_interpreter_call","outputs":[{"type":"logs","logs":"done"},{"type":"image","url":"https://example.test"}]}))]
#[case::image(json!({"type":"image_generation_call","result":"generated"}))]
#[case::mcp(json!({"type":"mcp_call","server_label":"server","name":"lookup","arguments":"{}","output":"done"}))]
fn output_items_round_trip(#[case] wire: Value) {
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
match &item {
ResponsesOutputItem::Message(message) => {
assert_eq!(message.role.as_deref(), Some("assistant"));
let Some(content) = &message.content else {
panic!("expected content")
};
let [
ResponsesContentPart::OutputText {
text,
annotations: Some(annotations),
..
},
] = content.as_slice()
else {
panic!("expected output text and annotations")
};
assert_eq!(text, "answer");
let [
litellm_llms_types::formats::responses::ResponsesAnnotation::UrlCitation(citation),
] = annotations.as_slice()
else {
panic!("expected URL citation")
};
assert_eq!(citation.url.as_deref(), Some("https://example.test"));
assert_eq!(citation.start_index, Some(0));
}
ResponsesOutputItem::FunctionCall(call) => {
assert_eq!(call.call_id.as_deref(), Some("call_1"));
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{\"query\":\"q\"}"));
}
ResponsesOutputItem::CustomToolCall(call) => {
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.input.as_deref(), Some("q"));
}
ResponsesOutputItem::Reasoning(reasoning) => {
assert_eq!(reasoning.encrypted_content.as_deref(), Some("opaque"));
let Some(summary) = &reasoning.summary else {
panic!("expected summary")
};
let [ResponsesContentPart::SummaryText { text, .. }] = summary.as_slice() else {
panic!("expected summary text")
};
assert_eq!(text, "summary");
}
ResponsesOutputItem::WebSearchCall(call) => {
let Some(ResponsesWebSearchAction::Search {
queries: Some(queries),
sources: Some(sources),
..
}) = &call.action
else {
panic!("expected search action")
};
assert_eq!(queries, &["q"]);
assert_eq!(sources[0].url.as_deref(), Some("https://example.test"));
}
ResponsesOutputItem::FileSearchCall(call) => {
assert_eq!(call.queries.as_deref(), Some(["q".to_owned()].as_slice()));
let Some(results) = &call.results else {
panic!("expected search results")
};
assert_eq!(results[0].file_id.as_deref(), Some("file_1"));
assert_eq!(results[0].score, Some(1.into()));
assert_eq!(
results[0].attributes.as_ref().unwrap()["custom"],
json!([1, null])
);
}
ResponsesOutputItem::CodeInterpreterCall(call) => {
let Some(outputs) = &call.outputs else {
panic!("expected code outputs")
};
let [
ResponsesCodeOutput::Logs { logs, .. },
ResponsesCodeOutput::Image { url, .. },
] = outputs.as_slice()
else {
panic!("expected logs and image")
};
assert_eq!(logs, "done");
assert_eq!(url, "https://example.test");
}
ResponsesOutputItem::ImageGenerationCall(call) => {
assert_eq!(call.result.as_deref(), Some("generated"))
}
ResponsesOutputItem::McpCall(call) => {
assert_eq!(call.server_label.as_deref(), Some("server"));
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{}"));
assert_eq!(call.output.as_deref(), Some("done"));
}
other => panic!("unexpected variant {other:?}"),
}
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
fn nested_event_response_exposes_typed_output_and_preserves_extensions() {
let wire = json!({
"id":"response_1",
"model":"example-model",
"status":"completed",
"output":[{"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{}","extension":true}],
"extension":{"nested":[1,null]}
});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(response.id.as_deref(), Some("response_1"));
assert_eq!(response.model.as_deref(), Some("example-model"));
let Some(output) = &response.output else {
panic!("expected typed output");
};
let [Recognized::Known(ResponsesOutputItem::FunctionCall(call))] = output.as_slice() else {
panic!("expected function call");
};
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{}"));
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
#[rstest]
#[case::empty(json!({}))]
#[case::partial(json!({"id":"response_1","output":[]}))]
fn nested_event_response_accepts_partial_metadata(#[case] wire: Value) {
round_trip::<ResponsesEventResponse>(wire);
}
#[rstest]
#[case::wrong_model(json!({"model":7}))]
#[case::wrong_status(json!({"status":false}))]
#[case::wrong_output(json!({"output":{}}))]
fn event_response_rejects_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesEventResponse>(wire).is_err());
}
#[rstest]
#[case::unknown_type(json!({"type":"future_item","id":"item_1","payload":[1,null]}))]
#[case::missing_tag(json!({"id":"item_1"}))]
#[case::malformed_known(json!({"type":"message","content":[{"type":"output_text","text":7}]}))]
fn event_response_keeps_unmodeled_output_items_beside_typed_ones(#[case] item: Value) {
let wire = json!({"output":[{"type":"function_call","name":"lookup"}, item.clone()]});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
let Some(
[
Recognized::Known(ResponsesOutputItem::FunctionCall(call)),
Recognized::Unrecognized(kept),
],
) = response.output.as_deref()
else {
panic!("expected one typed item and one preserved item");
};
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(kept, &item);
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
#[rstest]
#[case::find_in_page(
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}),
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"})
)]
#[case::legacy_find(
json!({"type":"find","url":"https://example.test","pattern":"needle"}),
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"})
)]
fn web_search_find_action_exposes_url_and_pattern(#[case] wire: Value, #[case] serialized: Value) {
let action: ResponsesWebSearchAction = serde_json::from_value(wire).unwrap();
let ResponsesWebSearchAction::FindInPage {
url,
pattern,
extra,
} = &action
else {
panic!("expected find_in_page action");
};
assert_eq!(url, "https://example.test");
assert_eq!(pattern, "needle");
assert!(extra.is_empty());
assert_eq!(serde_json::to_value(action).unwrap(), serialized);
}
#[rstest]
#[case::with_url(json!({"type":"open_page","url":"https://example.test"}), Some("https://example.test"))]
#[case::without_url(json!({"type":"open_page"}), None)]
fn web_search_open_page_url_is_optional(#[case] wire: Value, #[case] expected: Option<&str>) {
let action: ResponsesWebSearchAction = serde_json::from_value(wire.clone()).unwrap();
let ResponsesWebSearchAction::OpenPage { url, .. } = &action else {
panic!("expected open_page action");
};
assert_eq!(url.as_deref(), expected);
assert_eq!(serde_json::to_value(action).unwrap(), wire);
}
#[rstest]
#[case::find_pattern(json!({"type":"find_in_page","url":"u","pattern":false}))]
#[case::find_missing_url(json!({"type":"find_in_page","pattern":"p"}))]
#[case::open_page_url(json!({"type":"open_page","url":7}))]
fn web_search_action_rejects_malformed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesWebSearchAction>(wire).is_err());
}
#[rstest]
fn event_response_optional_fields_omit_missing_and_null() {
let response: ResponsesEventResponse = serde_json::from_value(json!({
"id":null,"model":null,"status":null,"output":null,"future":null
}))
.unwrap();
assert!(response.id.is_none());
assert!(response.model.is_none());
assert!(response.status.is_none());
assert!(response.output.is_none());
assert_eq!(
serde_json::to_value(response).unwrap(),
json!({"future":null})
);
}
#[rstest]
#[case::refusal(json!({"type":"refusal","refusal":"refused","future":null}), ResponsesContentPart::Refusal { refusal:"refused".into(), extra:serde_json::Map::from_iter([("future".into(), Value::Null)]) })]
#[case::reasoning(json!({"type":"reasoning_text","text":"reason"}), ResponsesContentPart::ReasoningText { text:"reason".into(), extra:Default::default() })]
fn content_parts_expose_typed_variants(
#[case] wire: Value,
#[case] expected: ResponsesContentPart,
) {
let content: ResponsesContentPart = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(content, expected);
assert_eq!(serde_json::to_value(content).unwrap(), wire);
}
#[rstest]
fn mcp_discovery_exposes_tools_and_nested_schemas() {
let wire = json!({
"type":"mcp_list_tools","id":"item_1","server_label":"tools",
"tools":[{"name":"lookup","description":"Look up a value","input_schema":{"type":"object","properties":{"query":{"type":"string"}}},"annotations":{"read_only":false},"future":null}],
"extension":[1,null]
});
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
let ResponsesOutputItem::McpListTools(discovery) = &item else {
panic!("expected discovery")
};
assert_eq!(discovery.server_label.as_deref(), Some("tools"));
let tool = &discovery.tools.as_ref().unwrap()[0];
assert_eq!(tool.name.as_deref(), Some("lookup"));
let Some(litellm_llms_types::json_schema::JsonSchema::Object(schema)) = &tool.input_schema
else {
panic!("expected tool schema")
};
assert!(schema.properties.as_ref().unwrap().contains_key("query"));
assert_eq!(tool.annotations, Some(json!({"read_only":false})));
assert_eq!(tool.extra["future"], Value::Null);
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
#[case::partial(json!({"server_label":"tools","tools":[]}))]
#[case::failure(json!({"server_label":"tools","error":"unavailable"}))]
fn mcp_discovery_accepts_partial_payloads(#[case] wire: Value) {
let discovery: litellm_llms_types::formats::responses::ResponsesMcpListTools =
serde_json::from_value(wire.clone()).unwrap();
assert_eq!(discovery.server_label.as_deref(), Some("tools"));
assert!(discovery.id.is_none());
assert_eq!(serde_json::to_value(discovery).unwrap(), wire);
}
#[rstest]
#[case::message(json!("unavailable"), ResponsesMcpError::Message("unavailable".into()))]
#[case::protocol(json!({"type":"mcp_protocol_error","code":-32600,"message":"invalid","extension":null}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpProtocolError {code:-32600,message:"invalid".into(),extra:serde_json::Map::from_iter([("extension".into(), Value::Null)])}))]
#[case::http(json!({"type":"http_error","code":503,"message":"unavailable"}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::HttpError {code:503,message:"unavailable".into(),extra:Default::default()}))]
#[case::tool(json!({"type":"mcp_tool_execution_error","content":{"arbitrary":[1,null]}}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpToolExecutionError {content:json!({"arbitrary":[1,null]}),extra:Default::default()}))]
fn mcp_call_errors_expose_typed_variants(#[case] wire: Value, #[case] expected: ResponsesMcpError) {
let error: ResponsesMcpError = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(error, expected);
assert_eq!(serde_json::to_value(&error).unwrap(), wire);
let call: ResponsesOutputItem =
serde_json::from_value(json!({"type":"mcp_call","error":wire})).unwrap();
let ResponsesOutputItem::McpCall(call) = call else {
panic!("expected call")
};
assert_eq!(call.error, Some(error));
}
#[rstest]
#[case::tools_not_array(json!({"type":"mcp_list_tools","tools":{}}))]
#[case::invalid_name(json!({"type":"mcp_list_tools","tools":[{"name":7}]}))]
#[case::invalid_schema(json!({"type":"mcp_list_tools","tools":[{"input_schema":{"properties":{"query":7}}}]}))]
#[case::invalid_error_code(json!({"type":"mcp_call","error":{"type":"http_error","code":"503","message":"unavailable"}}))]
#[case::missing_error_content(json!({"type":"mcp_call","error":{"type":"mcp_tool_execution_error"}}))]
#[case::unknown_error_tag(json!({"type":"mcp_call","error":{"type":"future"}}))]
fn mcp_output_rejects_malformed_known_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesOutputItem>(wire).is_err());
}
fn text(value: &str) -> Option<String> {
Some(value.to_owned())
}
fn program_caller() -> Option<ResponsesToolCaller> {
Some(ResponsesToolCaller::Program {
caller_id: "prog_1".into(),
extra: Default::default(),
})
}
fn direct_caller() -> Option<ResponsesToolCaller> {
Some(ResponsesToolCaller::Direct {
extra: Default::default(),
})
}
fn safety_check() -> ResponsesSafetyCheck {
ResponsesSafetyCheck {
id: text("sc_1"),
code: text("malicious_instructions"),
message: text("check"),
..Default::default()
}
}
#[rstest]
#[case::function_call(
json!({"type":"function_call","id":"fc_1","status":"completed","call_id":"call_1","name":"lookup","arguments":"{}","namespace":"ns","async":true,"caller":{"type":"program","caller_id":"prog_1"}}),
ResponsesOutputItem::FunctionCall(ResponsesFunctionCall {
id: text("fc_1"), status: text("completed"), call_id: text("call_1"), name: text("lookup"),
arguments: text("{}"), namespace: text("ns"), r#async: Some(true), caller: program_caller(), ..Default::default()
})
)]
#[case::custom_tool_call(
json!({"type":"custom_tool_call","call_id":"call_1","name":"lookup","input":"q","namespace":"ns","async":false,"caller":{"type":"direct"}}),
ResponsesOutputItem::CustomToolCall(ResponsesCustomToolCall {
call_id: text("call_1"), name: text("lookup"), input: text("q"), namespace: text("ns"),
r#async: Some(false), caller: direct_caller(), ..Default::default()
})
)]
#[case::image_generation_call(
json!({"type":"image_generation_call","id":"ig_1","status":"completed","result":"b64","action":"edit","background":"opaque","output_format":"webp","quality":"high","revised_prompt":"a cat","size":"1536x864"}),
ResponsesOutputItem::ImageGenerationCall(ResponsesImageGenerationCall {
id: text("ig_1"), status: text("completed"), result: text("b64"), action: text("edit"), background: text("opaque"),
output_format: text("webp"), quality: text("high"), revised_prompt: text("a cat"), size: text("1536x864"), ..Default::default()
})
)]
#[case::function_call_output_text(
json!({"type":"function_call_output","id":"fco_1","status":"completed","call_id":"call_1","output":"done","caller":{"type":"direct"},"created_by":"user_1","name":"lookup","namespace":"ns"}),
ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput {
id: text("fco_1"), status: text("completed"), call_id: text("call_1"), output: Some(ResponsesToolOutput::Text("done".into())),
caller: direct_caller(), created_by: text("user_1"), name: text("lookup"), namespace: text("ns"), ..Default::default()
})
)]
#[case::function_call_output_content(
json!({"type":"function_call_output","call_id":"call_1","output":[
{"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"explicit"}},
{"type":"input_image","detail":"low","file_id":"file_1","image_url":"https://example.test/i.png"},
{"type":"input_file","detail":"high","file_data":"data","file_id":"file_2","file_url":"https://example.test/f","filename":"f.pdf"}
]}),
ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput {
call_id: text("call_1"),
output: Some(ResponsesToolOutput::Content(vec![
ResponsesInputContent::InputText {
text: "t".into(),
prompt_cache_breakpoint: Some(PromptCacheBreakpoint { mode: PromptCacheMode::Explicit, extra: Default::default() }),
extra: Default::default(),
},
ResponsesInputContent::InputImage {
detail: "low".into(), file_id: text("file_1"), image_url: text("https://example.test/i.png"),
prompt_cache_breakpoint: None, extra: Default::default(),
},
ResponsesInputContent::InputFile {
detail: text("high"), file_data: text("data"), file_id: text("file_2"), file_url: text("https://example.test/f"),
filename: text("f.pdf"), prompt_cache_breakpoint: None, extra: Default::default(),
},
])),
..Default::default()
})
)]
#[case::custom_tool_call_output(
json!({"type":"custom_tool_call_output","id":"cto_1","status":"completed","call_id":"call_1","output":[{"type":"input_text","text":"t"}],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}),
ResponsesOutputItem::CustomToolCallOutput(ResponsesCustomToolCallOutput {
id: text("cto_1"), status: text("completed"), call_id: text("call_1"),
output: Some(ResponsesToolOutput::Content(vec![ResponsesInputContent::InputText {
text: "t".into(), prompt_cache_breakpoint: None, extra: Default::default(),
}])),
caller: program_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::computer_call(
json!({"type":"computer_call","id":"cu_1","status":"completed","call_id":"call_1",
"pending_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}],
"action":{"type":"click","button":"left","x":1,"y":2,"keys":["shift"]},
"actions":[
{"type":"double_click","x":3,"y":4},
{"type":"drag","path":[{"x":5,"y":6},{"x":7,"y":8}],"keys":["ctrl"]},
{"type":"keypress","keys":["enter"]},
{"type":"move","x":-1,"y":9},
{"type":"screenshot"},
{"type":"scroll","scroll_x":0,"scroll_y":-10,"x":11,"y":12},
{"type":"type","text":"hello"},
{"type":"wait"}
]}),
ResponsesOutputItem::ComputerCall(ResponsesComputerCall {
id: text("cu_1"), status: text("completed"), call_id: text("call_1"), pending_safety_checks: Some(vec![safety_check()]),
action: Some(ResponsesComputerAction::Click { button: "left".into(), x: 1, y: 2, keys: Some(vec!["shift".into()]), extra: Default::default() }),
actions: Some(vec![
ResponsesComputerAction::DoubleClick { x: 3, y: 4, keys: None, extra: Default::default() },
ResponsesComputerAction::Drag {
path: vec![
ResponsesCoordinate { x: 5, y: 6, extra: Default::default() },
ResponsesCoordinate { x: 7, y: 8, extra: Default::default() },
],
keys: Some(vec!["ctrl".into()]),
extra: Default::default(),
},
ResponsesComputerAction::Keypress { keys: vec!["enter".into()], extra: Default::default() },
ResponsesComputerAction::Move { x: -1, y: 9, keys: None, extra: Default::default() },
ResponsesComputerAction::Screenshot { extra: Default::default() },
ResponsesComputerAction::Scroll { scroll_x: 0, scroll_y: -10, x: 11, y: 12, keys: None, extra: Default::default() },
ResponsesComputerAction::Type { text: "hello".into(), extra: Default::default() },
ResponsesComputerAction::Wait { extra: Default::default() },
]),
..Default::default()
})
)]
#[case::computer_call_output(
json!({"type":"computer_call_output","id":"cuo_1","status":"completed","call_id":"call_1","output":{"type":"computer_screenshot","file_id":"file_1","image_url":"https://example.test/s.png"},"acknowledged_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}],"created_by":"user_1"}),
ResponsesOutputItem::ComputerCallOutput(ResponsesComputerCallOutput {
id: text("cuo_1"), status: text("completed"), call_id: text("call_1"),
output: Some(ResponsesComputerOutput::ComputerScreenshot { file_id: text("file_1"), image_url: text("https://example.test/s.png"), extra: Default::default() }),
acknowledged_safety_checks: Some(vec![safety_check()]), created_by: text("user_1"), ..Default::default()
})
)]
#[case::program(
json!({"type":"program","id":"prog_item","call_id":"prog_1","code":"run()","fingerprint":"fp"}),
ResponsesOutputItem::Program(ResponsesProgram {
id: text("prog_item"), call_id: text("prog_1"), code: text("run()"), fingerprint: text("fp"), ..Default::default()
})
)]
#[case::program_output(
json!({"type":"program_output","id":"po_1","status":"completed","call_id":"prog_1","result":"42"}),
ResponsesOutputItem::ProgramOutput(ResponsesProgramOutput {
id: text("po_1"), status: text("completed"), call_id: text("prog_1"), result: text("42"), ..Default::default()
})
)]
#[case::tool_search_call(
json!({"type":"tool_search_call","id":"ts_1","status":"completed","call_id":"call_1","arguments":{"query":["weather",null]},"execution":"server","created_by":"user_1"}),
ResponsesOutputItem::ToolSearchCall(ResponsesToolSearchCall {
id: text("ts_1"), status: text("completed"), call_id: text("call_1"), arguments: Some(json!({"query":["weather",null]})),
execution: text("server"), created_by: text("user_1"), ..Default::default()
})
)]
#[case::tool_search_output(
json!({"type":"tool_search_output","id":"tso_1","status":"completed","call_id":"call_1","execution":"client","tools":[{"type":"function","name":"lookup","parameters":null,"strict":true}],"created_by":"user_1"}),
ResponsesOutputItem::ToolSearchOutput(ResponsesToolSearchOutput {
id: text("tso_1"), status: text("completed"), call_id: text("call_1"), execution: text("client"),
tools: Some(vec![serde_json::Map::from_iter([
("type".into(), json!("function")), ("name".into(), json!("lookup")),
("parameters".into(), Value::Null), ("strict".into(), json!(true)),
])]),
created_by: text("user_1"), ..Default::default()
})
)]
#[case::additional_tools(
json!({"type":"additional_tools","id":"at_1","role":"developer","tools":[{"type":"local_shell"}]}),
ResponsesOutputItem::AdditionalTools(ResponsesAdditionalTools {
id: text("at_1"), role: text("developer"),
tools: Some(vec![serde_json::Map::from_iter([("type".into(), json!("local_shell"))])]), ..Default::default()
})
)]
#[case::compaction(
json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque","created_by":"user_1"}),
ResponsesOutputItem::Compaction(ResponsesCompaction {
id: text("cmp_1"), encrypted_content: text("opaque"), created_by: text("user_1"), ..Default::default()
})
)]
#[case::local_shell_call(
json!({"type":"local_shell_call","id":"ls_1","status":"completed","call_id":"call_1","action":{"type":"exec","command":["ls","-a"],"env":{"HOME":"/home/u"},"timeout_ms":1000,"user":"u","working_directory":"/tmp"}}),
ResponsesOutputItem::LocalShellCall(ResponsesLocalShellCall {
id: text("ls_1"), status: text("completed"), call_id: text("call_1"),
action: Some(ResponsesLocalShellAction::Exec {
command: vec!["ls".into(), "-a".into()], env: [("HOME".into(), "/home/u".into())].into(),
timeout_ms: Some(1000), user: text("u"), working_directory: text("/tmp"), extra: Default::default(),
}),
..Default::default()
})
)]
#[case::local_shell_call_output(
json!({"type":"local_shell_call_output","id":"lso_1","status":"completed","output":"{\"stdout\":\"x\"}"}),
ResponsesOutputItem::LocalShellCallOutput(ResponsesLocalShellCallOutput {
id: text("lso_1"), status: text("completed"), output: text("{\"stdout\":\"x\"}"), ..Default::default()
})
)]
#[case::shell_call_local(
json!({"type":"shell_call","id":"sh_1","status":"in_progress","call_id":"call_1","action":{"commands":["ls"],"max_output_length":100,"timeout_ms":500},"environment":{"type":"local"},"caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ShellCall(ResponsesShellCall {
id: text("sh_1"), status: text("in_progress"), call_id: text("call_1"),
action: Some(ResponsesShellAction { commands: Some(vec!["ls".into()]), max_output_length: Some(100), timeout_ms: Some(500), ..Default::default() }),
environment: Some(ResponsesShellEnvironment::Local { extra: Default::default() }),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::shell_call_container(
json!({"type":"shell_call","call_id":"call_1","environment":{"type":"container_reference","container_id":"cntr_1"}}),
ResponsesOutputItem::ShellCall(ResponsesShellCall {
call_id: text("call_1"),
environment: Some(ResponsesShellEnvironment::ContainerReference { container_id: "cntr_1".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::shell_call_output(
json!({"type":"shell_call_output","id":"sho_1","status":"completed","call_id":"call_1","max_output_length":100,"output":[
{"outcome":{"type":"exit","exit_code":2},"stdout":"out","stderr":"err","created_by":"user_1"},
{"outcome":{"type":"timeout"},"stdout":"","stderr":""}
],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}),
ResponsesOutputItem::ShellCallOutput(ResponsesShellCallOutput {
id: text("sho_1"), status: text("completed"), call_id: text("call_1"), max_output_length: Some(100),
output: Some(vec![
ResponsesShellOutputChunk {
outcome: Some(ResponsesShellOutcome::Exit { exit_code: 2, extra: Default::default() }),
stdout: text("out"), stderr: text("err"), created_by: text("user_1"), ..Default::default()
},
ResponsesShellOutputChunk {
outcome: Some(ResponsesShellOutcome::Timeout { extra: Default::default() }),
stdout: text(""), stderr: text(""), ..Default::default()
},
]),
caller: program_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::apply_patch_create(
json!({"type":"apply_patch_call","id":"ap_1","status":"completed","call_id":"call_1","operation":{"type":"create_file","path":"a.txt","diff":"+a"},"caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
id: text("ap_1"), status: text("completed"), call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::CreateFile { path: "a.txt".into(), diff: "+a".into(), extra: Default::default() }),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::apply_patch_delete(
json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"delete_file","path":"a.txt"}}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::DeleteFile { path: "a.txt".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::apply_patch_update(
json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"update_file","path":"a.txt","diff":"-a\n+b"}}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::UpdateFile { path: "a.txt".into(), diff: "-a\n+b".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::apply_patch_call_output(
json!({"type":"apply_patch_call_output","id":"apo_1","status":"failed","call_id":"call_1","output":"conflict","caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ApplyPatchCallOutput(ResponsesApplyPatchCallOutput {
id: text("apo_1"), status: text("failed"), call_id: text("call_1"), output: text("conflict"),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::mcp_approval_request(
json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}),
ResponsesOutputItem::McpApprovalRequest(ResponsesMcpApprovalRequest {
id: text("apr_1"), server_label: text("s"), name: text("n"), arguments: text("{}"), ..Default::default()
})
)]
#[case::mcp_approval_response(
json!({"type":"mcp_approval_response","id":"aprr_1","approval_request_id":"apr_1","approve":false,"reason":"denied"}),
ResponsesOutputItem::McpApprovalResponse(ResponsesMcpApprovalResponse {
id: text("aprr_1"), approval_request_id: text("apr_1"), approve: Some(false), reason: text("denied"), ..Default::default()
})
)]
fn documented_output_items_parse_into_typed_variants(
#[case] wire: Value,
#[case] expected: ResponsesOutputItem,
) {
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(item, expected);
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
#[case::compaction(json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque"}))]
#[case::approval(json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}))]
#[case::shell(json!({"type":"shell_call","call_id":"call_1","action":{"commands":["ls"]}}))]
fn event_response_types_documented_output_items(#[case] item: Value) {
let wire = json!({"output":[item.clone()]});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
let Some([Recognized::Known(known)]) = response.output.as_deref() else {
panic!("expected one typed item");
};
assert_eq!(
known,
&serde_json::from_value::<ResponsesOutputItem>(item).unwrap()
);
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
#[rstest]
#[case::unknown_caller(json!({"type":"function_call","caller":{"type":"future"}}))]
#[case::program_caller_missing_id(json!({"type":"shell_call","caller":{"type":"program"}}))]
#[case::untagged_caller(json!({"type":"custom_tool_call","caller":{}}))]
#[case::async_not_bool(json!({"type":"function_call","async":"yes"}))]
#[case::output_not_text_or_list(json!({"type":"function_call_output","output":{"text":"t"}}))]
#[case::unknown_input_content(json!({"type":"function_call_output","output":[{"type":"input_audio"}]}))]
#[case::input_text_missing_text(json!({"type":"custom_tool_call_output","output":[{"type":"input_text"}]}))]
#[case::input_image_missing_detail(json!({"type":"custom_tool_call_output","output":[{"type":"input_image","file_id":"f"}]}))]
#[case::bad_cache_mode(json!({"type":"custom_tool_call_output","output":[{"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"implicit"}}]}))]
#[case::unknown_computer_action(json!({"type":"computer_call","action":{"type":"teleport"}}))]
#[case::click_missing_button(json!({"type":"computer_call","action":{"type":"click","x":1,"y":2}}))]
#[case::click_fractional_x(json!({"type":"computer_call","action":{"type":"click","button":"left","x":1.5,"y":2}}))]
#[case::drag_point_missing_y(json!({"type":"computer_call","actions":[{"type":"drag","path":[{"x":1}]}]}))]
#[case::keypress_keys_not_list(json!({"type":"computer_call","actions":[{"type":"keypress","keys":"enter"}]}))]
#[case::safety_check_not_object(json!({"type":"computer_call","pending_safety_checks":["sc_1"]}))]
#[case::unknown_screenshot_tag(json!({"type":"computer_call_output","output":{"type":"screenshot"}}))]
#[case::untagged_screenshot(json!({"type":"computer_call_output","output":{"file_id":"f"}}))]
#[case::tool_not_object(json!({"type":"tool_search_output","tools":["lookup"]}))]
#[case::unknown_local_shell_action(json!({"type":"local_shell_call","action":{"type":"spawn","command":[],"env":{}}}))]
#[case::local_shell_missing_env(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"]}}))]
#[case::local_shell_env_value(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"],"env":{"A":1}}}))]
#[case::shell_commands_not_list(json!({"type":"shell_call","action":{"commands":"ls"}}))]
#[case::unknown_shell_environment(json!({"type":"shell_call","environment":{"type":"container_auto"}}))]
#[case::container_missing_id(json!({"type":"shell_call","environment":{"type":"container_reference"}}))]
#[case::unknown_shell_outcome(json!({"type":"shell_call_output","output":[{"outcome":{"type":"killed"}}]}))]
#[case::exit_missing_code(json!({"type":"shell_call_output","output":[{"outcome":{"type":"exit"}}]}))]
#[case::negative_max_output(json!({"type":"shell_call_output","max_output_length":-1}))]
#[case::unknown_patch_operation(json!({"type":"apply_patch_call","operation":{"type":"rename_file","path":"a"}}))]
#[case::update_missing_diff(json!({"type":"apply_patch_call","operation":{"type":"update_file","path":"a"}}))]
#[case::approve_not_bool(json!({"type":"mcp_approval_response","approve":"true"}))]
#[case::compaction_content_not_string(json!({"type":"compaction","encrypted_content":7}))]
#[case::program_code_not_string(json!({"type":"program","code":["run()"]}))]
fn output_items_reject_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesOutputItem>(wire).is_err());
}

36
litellm-rust/crates/llms/src/anthropic/beta_headers.rs Normal file
View file

@ -0,0 +1,36 @@
use litellm_http::request::{with_header, without_headers};
use litellm_llms_types::providers::anthropic::{BetaProvider, BetaSet};
use crate::{anthropic::common_utils::existing_betas, base_llm::auth::Headers};
const BETA_HEADER: &str = "anthropic-beta";
/// How the `anthropic-beta` header is treated before a request leaves for a host.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum BetaPolicy {
/// Send the header untouched and let the host decide.
Forward,
/// Keep only the betas the provider accepts, under the names it expects.
Filter(BetaProvider),
/// Remove every `anthropic-beta` header.
Drop,
}
impl BetaPolicy {
pub fn apply(self, headers: Headers) -> Headers {
let provider = match self {
Self::Forward => return headers,
Self::Drop => return without_headers(headers, &[BETA_HEADER]),
Self::Filter(provider) => provider,
};
let accepted: BetaSet = existing_betas(&headers)
.iter()
.filter_map(|beta| beta.on(provider))
.collect();
let headers = without_headers(headers, &[BETA_HEADER]);
if accepted.is_empty() {
return headers;
}
with_header(headers, BETA_HEADER, accepted.to_string())
}
}

1
litellm-rust/crates/llms/src/anthropic/mod.rs
View file

@ -1,3 +1,4 @@
pub mod beta_headers;
pub mod common_utils;
pub mod batches;

111
litellm-rust/crates/llms/tests/anthropic_beta_headers.rs Normal file
View file

@ -0,0 +1,111 @@
use litellm_llms::anthropic::beta_headers::BetaPolicy;
use litellm_llms_types::providers::anthropic::BetaProvider;
use rstest::rstest;
fn headers(pairs: &[(&str, &str)]) -> Vec<(String, String)> {
pairs
.iter()
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect()
}
#[rstest]
fn forward_keeps_headers_as_is() {
let input = headers(&[
(
"Anthropic-Beta",
"example-beta-2099-01-01,web-search-2025-03-05",
),
("x-api-key", "k"),
]);
assert_eq!(BetaPolicy::Forward.apply(input.clone()), input);
}
#[rstest]
fn drop_removes_every_casing() {
assert_eq!(
BetaPolicy::Drop.apply(headers(&[
("anthropic-beta", "a"),
("x-api-key", "k"),
("ANTHROPIC-BETA", "b"),
])),
headers(&[("x-api-key", "k")])
);
}
#[rstest]
#[case::bedrock(BetaProvider::Bedrock, Some("tool-search-tool-2025-10-19"))]
#[case::bedrock_mantle(BetaProvider::BedrockMantle, Some("tool-search-tool-2025-10-19"))]
#[case::vertex_ai(BetaProvider::VertexAi, Some("tool-search-tool-2025-10-19"))]
#[case::anthropic(BetaProvider::Anthropic, Some("advanced-tool-use-2025-11-20"))]
#[case::azure_ai(BetaProvider::AzureAi, Some("advanced-tool-use-2025-11-20"))]
#[case::databricks(BetaProvider::Databricks, Some("advanced-tool-use-2025-11-20"))]
#[case::bedrock_converse(BetaProvider::BedrockConverse, None)]
fn filter_renames_per_host(#[case] provider: BetaProvider, #[case] expected_beta: Option<&str>) {
let expected = match expected_beta {
Some(beta) => headers(&[("x-api-key", "k"), ("anthropic-beta", beta)]),
None => headers(&[("x-api-key", "k")]),
};
assert_eq!(
BetaPolicy::Filter(provider).apply(headers(&[
("Anthropic-Beta", "advanced-tool-use-2025-11-20"),
("x-api-key", "k"),
])),
expected
);
}
#[rstest]
#[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")]
#[case::anthropic(BetaProvider::Anthropic, "bash_20241022")]
#[case::vertex_ai(BetaProvider::VertexAi, " , ")]
fn filter_removes_header_when_nothing_survives(
#[case] provider: BetaProvider,
#[case] beta_value: &str,
) {
assert_eq!(
BetaPolicy::Filter(provider).apply(headers(&[
("x-api-key", "k"),
("anthropic-beta", beta_value)
])),
headers(&[("x-api-key", "k")])
);
}
#[rstest]
#[case::anthropic(
BetaProvider::Anthropic,
"web-search-2025-03-05",
"oauth-2025-04-20,web-search-2025-03-05,example-beta-2099-01-01",
"oauth-2025-04-20,web-search-2025-03-05"
)]
#[case::vertex_ai_renames_and_deduplicates(
BetaProvider::VertexAi,
"advanced-tool-use-2025-11-20",
"tool-search-tool-2025-10-19",
"tool-search-tool-2025-10-19"
)]
fn filter_deduplicates_and_sorts(
#[case] provider: BetaProvider,
#[case] first_beta: &str,
#[case] second_beta: &str,
#[case] expected_beta: &str,
) {
assert_eq!(
BetaPolicy::Filter(provider).apply(headers(&[
("ANTHROPIC-BETA", first_beta),
("x-api-key", "k"),
("anthropic-beta", second_beta),
])),
headers(&[("x-api-key", "k"), ("anthropic-beta", expected_beta)])
);
}
#[rstest]
#[case::forward(BetaPolicy::Forward)]
#[case::drop(BetaPolicy::Drop)]
#[case::filter(BetaPolicy::Filter(BetaProvider::Bedrock))]
fn headers_without_a_beta_are_untouched(#[case] policy: BetaPolicy) {
let input = headers(&[("x-api-key", "k"), ("anthropic-version", "2023")]);
assert_eq!(policy.apply(input.clone()), input);
}

13
litellm-rust/crates/python-bridge/src/lib.rs
View file

@ -17,6 +17,10 @@ mod tokenizer;
#[pymodule(gil_used = true)]
mod _native {
#[pymodule_export]
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
use pyo3::{prelude::*, types::PyModule};
#[cfg(feature = "panic-test")]
#[pymodule_export]
use crate::diagnostics::_panic_for_test;
@ -29,9 +33,7 @@ mod _native {
#[pymodule_export]
use crate::routes::audio_transcription::{atranscription, transcription};
#[pymodule_export]
use crate::routes::chat_completions::{
achat_completions, acompletion, chat_completions, completion,
};
use crate::routes::chat_completions::{acompletion, completion};
#[pymodule_export]
use crate::routes::embeddings::{aembedding, embedding};
#[pymodule_export]
@ -51,9 +53,6 @@ mod _native {
use crate::tokenizer::HuggingFaceEncoding;
#[pymodule_export]
use crate::tokenizer::Tokenizer;
#[pymodule_export]
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
use pyo3::{prelude::*, types::PyModule};
#[pymodule_init]
fn init(module: &Bound<'_, PyModule>) -> PyResult<()> {
@ -101,8 +100,6 @@ mod tests {
"atranscription",
"messages",
"amessages",
"chat_completions",
"achat_completions",
"completion",
"acompletion",
"responses",

19
litellm-rust/crates/python-bridge/src/marshal.rs
View file

@ -19,13 +19,6 @@ pub(crate) struct RouteOptions {
pub(crate) timeout: Option<Duration>,
}
pub(crate) fn messages_argument(value: &Bound<'_, PyAny>) -> PyResult<Vec<Value>> {
match from_py_argument(value)? {
Value::Array(values) => Ok(values),
_ => Err(PyValueError::new_err("messages must be a list")),
}
}
fn required_object(name: &'static str, value: Value) -> PyResult<Map<String, Value>> {
match value {
Value::Object(values) => Ok(values),
@ -449,18 +442,6 @@ module.__getattr__ = fail
fn argument_converters_keep_nested_values_and_accept_explicit_none() {
Python::initialize();
Python::attach(|py| {
let messages = py
.eval(
c"[{'role': 'user', 'content': [{'type': 'text', 'text': 'hi'}]}]",
None,
None,
)
.unwrap();
assert_eq!(
Value::Array(messages_argument(&messages).unwrap()),
json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}])
);
let params = py.eval(c"{'temperature': 0.2}", None, None).unwrap();
assert_eq!(
optional_object("optional_params", &params).unwrap(),

4
litellm-rust/crates/python-bridge/src/routes/AGENTS.md
View file

@ -11,3 +11,7 @@ The host driver owns sequencing and terminal events; the bridge supplies fallibl
Use the shared `run_public_call` boundary with hooks supplied by bridge composition. `callbacks-legacy-python` owns legacy argument sharing and `Logging` dispatch behind `PublicCall` and `LegacyLogging`. Route bindings supply `callbacks-legacy-python::LoggingOperation` when composing legacy logging and may retain the request needed for projection, but must not duplicate the legacy callback contract
Regression tests must observe that an unstarted call does no setup, hook and preflight rewrites affect resource configuration, setup failures reach the selected failure handler once, and provider work is not replayed. Retain existing read-point and object-identity guarantees while changing setup timing
## Layout
`messages/` is the reference shape. Each lifecycle route is a folder: `mod.rs` holds `run_<route>` and the thin sync and async `#[pyfunction]` entrypoints that take `NativeCall` and call it, `host.rs` holds the `PythonBinding` host, and anything else route-specific (projection, error mapping, extra pyclasses) gets its own sibling file. Scaffolded routes without a lifecycle (embeddings, audio transcription) stay a single `<route>.rs`

8
litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs
View file

@ -5,6 +5,8 @@ use litellm_inference_transcription::{
use pyo3::prelude::*;
use serde_json::{Map, Value};
use super::NativeCall;
use crate::{
errors::route_error_to_pyerr,
marshal::{RouteOptions, optional_object_field, required_field, value_route_options},
@ -40,8 +42,7 @@ async fn execute(
}
#[pyfunction]
pub(crate) fn transcription(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
pub(crate) fn transcription(py: Python<'_>, call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
let audio: Value =
litellm_host_python::from_py_argument(&required_field(&call.bound, "audio")?)?;
let options = value_route_options(&call.bound)?;
@ -59,9 +60,8 @@ pub(crate) fn transcription(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<
#[pyfunction]
pub(crate) fn atranscription<'py>(
py: Python<'py>,
call: Bound<'py, PyAny>,
call: NativeCall<'py>,
) -> PyResult<Bound<'py, PyAny>> {
let call = super::NativeCall::extract(&call)?;
let audio: Value =
litellm_host_python::from_py_argument(&required_field(&call.bound, "audio")?)?;
let options = value_route_options(&call.bound)?;

150
litellm-rust/crates/python-bridge/src/routes/chat_completions.rs
View file

@ -1,150 +0,0 @@
mod host;
use pyo3::types::{PyDict, PyTuple};
use crate::execution::{run_async, run_sync};
use litellm_inference_chat::{ChatCompletionsRoute, Error, types::ChatCompletionsRequest};
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
use pyo3::prelude::*;
use serde_json::{Map, Value};
use crate::{
errors::route_error_to_pyerr,
marshal::{
RouteOptions, messages_argument, optional_object_field, required_field, value_route_options,
},
};
async fn execute(
http: Result<litellm_http::Client, litellm_http::Error>,
secrets: std::sync::Arc<dyn litellm_secrets::source::SecretSource>,
messages: Vec<Value>,
optional_params: Map<String, Value>,
options: RouteOptions,
) -> Result<ChatCompletionsResponse, Error> {
let RouteOptions {
model,
api_key,
api_base,
custom_llm_provider,
extra_headers,
timeout,
} = options;
ChatCompletionsRoute::new(http?, crate::http::resources().auth.clone(), secrets)
.execute(
ChatCompletionsRequest {
model: &model,
messages: Value::Array(messages),
optional_params,
api_key: api_key.as_deref(),
api_base: api_base.as_deref(),
custom_llm_provider: custom_llm_provider.as_deref(),
extra_headers,
timeout,
},
&(),
None,
)
.await
}
#[pyfunction]
pub(crate) fn chat_completions(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
let messages: Vec<Value> = messages_argument(&required_field(&call.bound, "messages")?)?;
let optional_params =
optional_object_field(&call.bound, "optional_params")?.unwrap_or_default();
let options = value_route_options(&call.bound)?;
let http = crate::http::provider_client(py, &call.kwargs, false)?;
let secrets = crate::secrets::source(py)?;
run_sync(
py,
execute(http, secrets, messages, optional_params, options),
route_error_to_pyerr,
)
}
#[pyfunction]
pub(crate) fn achat_completions<'py>(
py: Python<'py>,
call: Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
let call = super::NativeCall::extract(&call)?;
let messages: Vec<Value> = messages_argument(&required_field(&call.bound, "messages")?)?;
let optional_params =
optional_object_field(&call.bound, "optional_params")?.unwrap_or_default();
let options = value_route_options(&call.bound)?;
let http = crate::http::provider_client(py, &call.kwargs, true)?;
let secrets = crate::secrets::source(py)?;
run_async(
py,
execute(http, secrets, messages, optional_params, options),
route_error_to_pyerr,
)
}
fn run_public(
py: Python<'_>,
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
asynchronous: bool,
) -> PyResult<Py<PyAny>> {
use super::inference::InferenceHost;
use litellm_callbacks_legacy_python::LoggingOperation;
let host = InferenceHost::new(
request.clone().unbind(),
"litellm.rust_bridge.chat_completions.route_host",
);
let cache_call_type = if asynchronous {
"acompletion"
} else {
"completion"
};
crate::cache::admit_native(py, &kwargs, cache_call_type)?;
let (arguments, hooks) = crate::routes::call_hooks(
py,
LoggingOperation::Completion,
&request,
&args,
&kwargs,
asynchronous,
)?;
crate::routes::run_public_call(
py,
arguments,
move |py, arguments, request| {
let route = ChatCompletionsRoute::new(
crate::http::provider_client(py, arguments, asynchronous)?
.map_err(crate::http::client_error)?,
crate::http::resources().auth.clone(),
crate::secrets::source(py)?,
);
let (cache, cache_options) =
crate::cache::configured_native(py, arguments, cache_call_type)?;
let route = match cache {
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
cache,
litellm_cache_response::CacheScope::Shared,
)),
None => route,
};
Ok(route.machine(request, cache_options.policy))
},
host::ChatCompletionsPythonHost(host),
hooks,
asynchronous,
)
}
#[pyfunction]
pub(crate) fn completion(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_public(py, call.bound.into_any(), call.args, call.kwargs, false)
}
#[pyfunction]
pub(crate) fn acompletion(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_public(py, call.bound.into_any(), call.args, call.kwargs, true)
}

2
litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs
View file

@ -1,6 +1,6 @@
use std::convert::Infallible;
use super::super::inference::InferenceHost;
use crate::routes::inference::InferenceHost;
use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned};
use litellm_inference_chat::{Error, route::ChatCompletions, types::ChatCompletionsCall};
use pyo3::{

71
litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs Normal file
View file

@ -0,0 +1,71 @@
mod host;
use std::sync::Arc;
use host::ChatCompletionsPythonHost;
use litellm_auth::AuthServices;
use litellm_cache_response::{CachePolicy, ScopedCache};
use litellm_callbacks_legacy_python::LoggingOperation;
use litellm_host::{call::HostedMachine, protocol::Protocol};
use litellm_inference_chat::{ChatCompletionsRoute, route::ChatCompletions};
use litellm_secrets::source::SecretSource;
use pyo3::prelude::*;
use super::{
NativeCall,
inference::{InferenceHost, InferenceRoute, run_inference},
};
fn run_chat_completions(
py: Python<'_>,
call: NativeCall<'_>,
asynchronous: bool,
) -> PyResult<Py<PyAny>> {
let host = InferenceHost::new(
call.bound.clone().unbind(),
"litellm.rust_bridge.chat_completions.route_host",
);
run_inference::<ChatCompletionsRoute, _>(
py,
call,
asynchronous,
ChatCompletionsPythonHost(host),
)
}
#[pyfunction]
pub(crate) fn completion(py: Python<'_>, call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
run_chat_completions(py, call, false)
}
#[pyfunction]
pub(crate) fn acompletion(py: Python<'_>, call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
run_chat_completions(py, call, true)
}
impl InferenceRoute for ChatCompletionsRoute {
type Protocol = ChatCompletions;
const OPERATION: LoggingOperation = LoggingOperation::Completion;
const SYNC_CALL_TYPE: &'static str = "completion";
const ASYNC_CALL_TYPE: &'static str = "acompletion";
fn new(
http: litellm_http::Client,
auth: Arc<AuthServices>,
secrets: Arc<dyn SecretSource>,
) -> Self {
Self::new(http, auth, secrets)
}
fn with_cache(self, cache: ScopedCache) -> Self {
self.with_cache(cache)
}
fn machine(
self,
call: <ChatCompletions as Protocol>::Request,
policy: CachePolicy,
) -> HostedMachine<ChatCompletions> {
self.machine(call, policy)
}
}

Some files were not shown because too many files have changed in this diff Show more