scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

chore: merge main into litellm_remove_lit002_dict_ban

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-10-09 02:11:27 +00:00
commit 398a8b1224
762 changed files with 63130 additions and 29512 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

140
.circleci/config.yml
View file

@ -1095,50 +1095,6 @@ jobs:
- google_generate_content_endpoint_coverage.xml
- google_generate_content_endpoint_coverage
llm_responses_api_testing:
docker:
- *python312_image
working_directory: ~/project
resource_class: large
environment:
REQUEST_TIMEOUT: "180"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- restore_cache:
keys:
- v1-uv-cache-{{ checksum "uv.lock" }}
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- save_cache:
paths:
- ~/.cache/uv
key: v1-uv-cache-{{ checksum "uv.lock" }}
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-v \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 8 \
--reruns 1 --only-rerun Timeout"
no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
ocr_testing:
docker:
- *python312_image
@ -2074,98 +2030,6 @@ jobs:
# Store test results
- store_test_results:
path: test-results
proxy_spend_accuracy_tests:
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- start_postgres
- start_redis
- start_fake_openai_endpoint
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# Point the proxy at the job-local Redis (start_redis) instead of the
# shared remote Redis. The Redis transaction buffer uses a single
# global pod-lock key (cronjob_lock:db_spend_update_job) and a single
# global buffer list (litellm_spend_update_buffer); sharing those
# across concurrent CI pipelines causes spend flushes to stall or
# land in the wrong DB, which is what makes this test flaky.
command: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=host.docker.internal \
-e REDIS_PORT=6379 \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
-e PROXY_BATCH_WRITE_AT=2 \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000
- run:
name: Start outputting logs
command: docker logs -f my-app
background: true
- wait_for_service:
url: http://localhost:4000
timeout: "300"
- run:
name: Run tests
command: |
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-vv \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- store_test_results:
path: test-results
- run:
name: Stop and remove first container
when: always
command: |
docker stop my-app
docker rm my-app
docker stop redis-cache
docker rm redis-cache
proxy_multi_instance_tests:
machine:
image: ubuntu-2204:2024.04.1
@ -3591,9 +3455,6 @@ workflows:
- proxy_logging_guardrails_model_info_tests:
requires:
- build_docker_database_image
- proxy_spend_accuracy_tests:
requires:
- build_docker_database_image
- proxy_multi_instance_tests:
requires:
- build_docker_database_image
@ -3611,7 +3472,6 @@ workflows:
- realtime_translation_testing
- guardrails_testing
- google_generate_content_endpoint_testing
- llm_responses_api_testing
- ocr_testing
- search_testing
- batches_testing

4
.github/workflows/_test-unit-base.yml vendored
View file

@ -143,9 +143,9 @@ jobs:
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"

65
.github/workflows/publish-basedpyright-base-counts.yml vendored
View file

@ -1,65 +0,0 @@
name: Publish basedpyright base counts
# Every commit on main can become a future merge-base.
# Publishing its per-rule basedpyright counts as an artifact lets
# scripts/type_check_gate.py download them in seconds instead of paying a
# 60-110s second basedpyright pass on every fresh worktree or moved merge-base.
# No concurrency group on purpose: runs must never cancel each other, because
# every sha's artifact matters (any of them can become a merge-base).
on:
push:
branches:
- main
workflow_dispatch:
inputs:
ref:
description: "Ref to compute and publish base counts for (defaults to the workflow run's commit)"
required: false
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ inputs.ref || github.sha }}
clean: true
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
# The gate provisions its own measurement env (.venv-typecheck: a frozen
# uv sync of its canonical dependency groups plus a generated Prisma
# client), so no install step here can drift from what local runs measure.
- name: Emit basedpyright counts for HEAD
run: |
python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts"
counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json)
echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
- name: Upload counts artifact
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: ${{ env.COUNTS_ARTIFACT_NAME }}
path: ${{ runner.temp }}/basedpyright-counts/
if-no-files-found: error

96
.github/workflows/publish-lint-base-counts.yml vendored Normal file
View file

@ -0,0 +1,96 @@
name: Publish lint base counts
# Every commit on main can become a future merge-base.
# Publishing its per-rule counts for each lint gate (strict ruff, type discipline,
# test quality, basedpyright) as an artifact lets the gates download them through
# scripts/lint_base_counts.py in seconds instead of scanning the merge-base tree in
# a throwaway worktree on every fresh checkout or moved merge-base.
# No concurrency group on purpose: runs must never cancel each other, because
# every sha's artifact matters (any of them can become a merge-base).
on:
push:
branches:
- main
workflow_dispatch:
inputs:
ref:
description: "Ref to compute and publish base counts for (defaults to the workflow run's commit)"
required: false
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
include:
- checker: ruff-strict
script: scripts/ruff_strict_gate.py
- checker: type-discipline
script: scripts/type_discipline_gate.py
- checker: test-quality
script: scripts/test_quality_gate.py
- checker: basedpyright
script: scripts/type_check_gate.py
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ inputs.ref || github.sha }}
clean: true
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
if: matrix.checker == 'basedpyright'
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
if: matrix.checker == 'basedpyright'
uses: ./.github/actions/cache-prisma-binaries
# The three source scanners only need the pinned dev tools (ruff and the
# stdlib checkers), the same versions test-linting.yml's lint job runs.
- name: Install the dev tools
if: matrix.checker != 'basedpyright'
run: |
uv sync --frozen --only-group dev --no-install-project
- name: Emit ${{ matrix.checker }} counts for HEAD
if: matrix.checker != 'basedpyright'
run: |
uv run --no-sync python ${{ matrix.script }} --emit-counts-dir "$RUNNER_TEMP/lint-counts"
# The basedpyright gate provisions its own measurement env (.venv-typecheck:
# a frozen uv sync of its canonical dependency groups plus a generated Prisma
# client), so no install step here can drift from what local runs measure.
- name: Emit basedpyright counts for HEAD
if: matrix.checker == 'basedpyright'
run: |
python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/lint-counts"
- name: Name the counts artifact
run: |
counts_file=$(ls "$RUNNER_TEMP"/lint-counts/${{ matrix.checker }}-counts-*.json)
echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
- name: Upload counts artifact
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: ${{ env.COUNTS_ARTIFACT_NAME }}
path: ${{ runner.temp }}/lint-counts/
if-no-files-found: error

3
.github/workflows/test-code-quality.yml vendored
View file

@ -89,6 +89,9 @@ jobs:
- name: test_workflow_job_name_collisions
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py
- name: test_unit_passed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_unit_passed_gate.py
- name: test_e2e_changed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py

54
.github/workflows/test-linting.yml vendored
View file

@ -17,9 +17,9 @@ jobs:
lint:
runs-on: ubuntu-latest
timeout-minutes: 15
# actions: read lets scripts/type_check_gate.py download the base-counts
# artifact published by publish-basedpyright-base-counts.yml instead of
# re-running basedpyright over the merge-base tree.
# actions: read lets the four lint gates download the base-counts artifacts
# published by publish-lint-base-counts.yml instead of re-scanning the
# merge-base tree in a throwaway worktree.
permissions:
contents: read
pull-requests: read
@ -144,18 +144,24 @@ jobs:
run: |
uv run --no-sync ruff check --config ruff-tests.toml tests
- name: Check strict-rule budget (delta vs base)
- name: Check strict ruff rules (delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
- name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
- name: Check type discipline (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
- name: Check test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs base)
- name: Check test quality (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA"
@ -164,7 +170,7 @@ jobs:
run: |
uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Check basedpyright budget (delta vs base)
- name: Check basedpyright (delta vs merge-base counts)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
@ -206,40 +212,6 @@ jobs:
run: |
uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Intentionally NON-GATING. This job turns red when a *-budget.json ceiling is
# raised (or a rule/budget is dropped) so a loosening is obvious in review, but it
# must be kept OUT of the branch-protection required-checks list so a justified
# bump can still be merged by a human who has seen and accepted the red.
budget-ratchet:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 1
persist-credentials: false
- name: Fetch ratchet base
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
retry git fetch --no-tags --depth=1 origin "$BASE_SHA"
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Ratchet check (budgets may only decrease; non-gating)
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
python scripts/budget_ratchet_check.py --base "$BASE_SHA"
secret-scan:
runs-on: ubuntu-latest
timeout-minutes: 5

64
.github/workflows/test-mcp-dependency-resolution.yml vendored
View file

@ -13,6 +13,9 @@ on:
- "rust-toolchain.toml"
- "litellm-rust/**"
- "litellm/__init__.py"
- "litellm/_version.py"
- "litellm/_lazy_imports_registry.py"
- "scripts/build_core_distribution.py"
- "litellm/proxy/proxy_server.py"
- "litellm/**/*mcp*"
- "litellm/**/*mcp*/**"
@ -86,6 +89,7 @@ jobs:
uv pip check --python ".venv-$extra"
if [ "$extra" = core ]; then
checker=("$GITHUB_WORKSPACE/tests/base_sdk_tests/check_base_sdk_install.py")
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-$extra/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_sdk_http.py")
else
checker=("$GITHUB_WORKSPACE/scripts/check_mcp_sdk_install.py" --extra "$extra")
fi
@ -115,3 +119,63 @@ jobs:
fi
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-lowest-$extra/bin/python" "${checker[@]}")
done
core-distribution:
permissions:
contents: read
id-token: write
runs-on: ubuntu-latest
timeout-minutes: 30
env:
UV_PYTHON: ${{ matrix.python-version }}
LITELLM_LOCAL_MODEL_COST_MAP: "True"
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ matrix.python-version }}
- uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- uses: ./.github/actions/cache-cargo-build
- name: Build the core wheel and source distribution
run: >-
uv run --no-project --with coverage==7.14.0 python -m coverage run --rcfile=/dev/null
--branch --include="*/scripts/build_core_distribution.py"
scripts/build_core_distribution.py --out-dir dist/core
- name: Verify metadata, resources, and independent source rebuild
env:
CORE_DISTRIBUTION_DIR: ${{ github.workspace }}/dist/core
run: >-
uv run --no-project --with pytest==9.0.3 --with pytest-cov==5.0.0 --with coverage==7.14.0 --with 'tomli==2.4.1; python_version < "3.11"'
python -m pytest tests/base_sdk_tests/test_core_distribution.py -v
--cov-config=/dev/null --cov=scripts.build_core_distribution --cov-branch --cov-append --cov-report=
- name: Verify isolated core installations
run: |
wheel=$(realpath dist/core/litellm_core-*.whl)
for resolution in highest lowest-direct; do
uv venv --python ${{ matrix.python-version }} ".venv-core-$resolution"
uv pip install --python ".venv-core-$resolution" --resolution "$resolution" "$wheel"
uv pip check --python ".venv-core-$resolution"
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-$resolution/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_base_sdk_install.py" --profile core)
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-$resolution/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_sdk_http.py")
done
uv pip install --python .venv-core-highest coverage==7.14.0
.venv-core-highest/bin/python -I -m coverage run --rcfile=/dev/null --append --branch --include="*/tests/base_sdk_tests/check_base_sdk_install.py" tests/base_sdk_tests/check_base_sdk_install.py --profile core
uv pip install --python .venv-core-highest boto3 tokenizers huggingface-hub
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-highest/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_base_sdk_install.py" --profile dependencies)
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-highest/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_sdk_http.py")
uv run --no-project --with coverage==7.14.0 python -m coverage xml --rcfile=/dev/null -o coverage-core.xml
- name: Upload core packaging coverage
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
files: coverage-core.xml
flags: core-packaging
fail_ci_if_error: false

3
.github/workflows/test-rust.yml vendored
View file

@ -98,6 +98,9 @@ jobs:
rust-test:
runs-on: ubuntu-latest
timeout-minutes: 30
# Trim debuginfo to keep the job's target dir within the runner's disk
env:
CARGO_PROFILE_DEV_DEBUG: line-tables-only
defaults:
run:
working-directory: litellm-rust

223
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -1,223 +0,0 @@
name: "Unit Tests: Proxy DB Operations"
on:
pull_request:
branches:
- main
- "litellm_**"
push:
branches:
- main
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# Semantic matrix: each shard groups tests by concern (auth, server, logging, …)
# rather than alphabetical letter ranges. Adding a new test file means adding it
# to whichever group it belongs to, not reshuffling slices.
#
# Design targets:
# * Every shard runs in <= 7 minutes of wall-clock on the default runner.
# Most of a shard's time is pytest plugin load + xdist worker imports +
# pytest-cov instrumentation, not the tests themselves. Keeping per-shard
# work low and matching worker count to runner cores is what controls it.
# * `timeout` bounds the pytest step only. Checkout, dependency install, and
# Prisma client generation draw on a separate allowance in the base
# workflow, so slow setup shows up as a slow job rather than as a
# cancelled shard whose tests were passing.
# * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores
# oversubscribes 2x and workers fight for CPU during their cold-start
# imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective).
# * test_key_generate_prisma.py stays serial (workers=0) — it has event-loop
# conflicts with the logging worker when run in parallel.
# * test_proxy_utils.py runs as a single shard with --dist=worksteal so
# xdist balances its 188 parametrized cases across workers instead of
# pinning the whole file to one worker (the default --dist=loadscope
# behavior for single-file targets).
jobs:
lens-python-310:
name: Lens Python 3.10
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
python-version: "3.10"
test-path: tests/unit/proxy/lens/test_inference.py
workers: 0
reruns: 0
timeout-minutes: 5
artifact-name: lens-python-310
# Fast guard — fails the workflow when a test directory or file inside a sharded
# tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
# so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list
# and reads the same test-path keys the coverage census does.
assert-shard-coverage:
runs-on: ubuntu-latest
timeout-minutes: 2
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Assert every test directory and file is claimed by a shard
run: python3 .github/scripts/assert_ci_coverage.py --shards
proxy-db:
needs: assert-shard-coverage
# Display only the semantic shard name in the checks UI instead of GHA's
# default "proxy-db (key-generation, tests/unit/proxy/…, 0, loadscope, 20)"
# which includes every matrix field and gets truncated past the test-path.
name: ${{ matrix.test-group }}
permissions:
contents: read
id-token: write
pull-requests: write
strategy:
fail-fast: false
matrix:
include:
# Must run serially — event-loop conflict with the logging worker.
- test-group: key-generation
test-path: >-
tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
workers: 0
dist: loadscope
timeout: 20
# ---- auth: split into 2 shards ----
- test-group: auth-checks
test-path: >-
tests/unit/proxy/auth/test_auth_checks.py
tests/unit/proxy/auth/test_user_api_key_auth.py
tests/unit/proxy/test_credential_slot_registry.py
tests/unit/proxy/test_deprecated_key_grace_period.py
workers: 4
dist: loadscope
timeout: 15
- test-group: jwt-and-keys
test-path: >-
tests/unit/proxy/auth/test_jwt.py
tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
tests/unit/proxy/test_proxy_custom_auth.py
workers: 4
dist: loadscope
timeout: 15
# ---- test_proxy_utils.py, single shard, worksteal distribution ----
- test-group: proxy-utils
test-path: >-
tests/unit/proxy/test_proxy_utils.py
workers: 4
dist: worksteal
timeout: 15
# ---- proxy server: split into 2 shards ----
- test-group: proxy-server-core
test-path: >-
tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py
tests/unit/proxy/test__lazy_features.py
tests/unit/proxy/test_aproxy_startup.py
tests/unit/proxy/test_proxy_server.py
workers: 4
dist: loadscope
timeout: 15
- test-group: proxy-runtime
test-path: >-
tests/unit/proxy/auth/test_multipart_bypass_repro.py
tests/unit/proxy/auth/test_proxy_routes.py
tests/unit/proxy/middleware/test_request_size_limit_middleware.py
tests/unit/proxy/test_proxy_config_unit_test.py
tests/unit/proxy/test_proxy_token_counter.py
tests/unit/proxy/test_server_root_path.py
workers: 4
dist: loadscope
timeout: 15
# ---- logging: split into 2 shards ----
- test-group: custom-logging
test-path: >-
tests/proxy_unit_tests/test_proxy_custom_logger.py
tests/unit/proxy/test_custom_callback_input.py
tests/unit/proxy/test_custom_logger_s3_gcs.py
workers: 4
dist: loadscope
timeout: 15
- test-group: logging-misc
test-path: >-
tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
tests/unit/proxy/spend_tracking/test_search_api_logging.py
tests/unit/proxy/test_proxy_reject_logging.py
workers: 4
dist: loadscope
timeout: 15
- test-group: db-and-spend
test-path: >-
tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
tests/unit/proxy/db/test_update_daily_tag_spend.py
tests/unit/proxy/test_db_schema_changes.py
tests/unit/proxy/test_prisma_client_backoff_retry.py
tests/unit/proxy/test_update_spend.py
tests/unit/skills/test_skills_db.py
workers: 4
dist: loadscope
timeout: 15
# ---- guardrails + budget + hooks: split into 2 ----
- test-group: guardrails-hooks
test-path: >-
tests/unit/proxy/hooks/test_banned_keyword_list.py
tests/unit/proxy/test_proxy_setting_guardrails.py
tests/unit/proxy/test_unit_test_proxy_hooks.py
workers: 4
dist: loadscope
timeout: 15
- test-group: budgets
test-path: >-
tests/unit/proxy/auth/test_default_end_user_budget_simple.py
tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
tests/unit/proxy/test_zero_cost_model_budget_bypass.py
workers: 4
dist: loadscope
timeout: 15
- test-group: endpoints-and-responses
test-path: >-
tests/proxy_unit_tests/test_proxy_exception_mapping.py
tests/unit/proxy/lens
tests/unit/proxy/auth/test_models_fallback_endpoint.py
tests/unit/proxy/common_utils/test_check_batch_cost.py
tests/unit/proxy/common_utils/test_check_responses_cost.py
tests/unit/proxy/common_utils/test_realtime_cache.py
tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
tests/unit/proxy/response_polling
tests/unit/proxy/test_custom_tokenizer_bug.py
tests/unit/proxy/test_get_favicon.py
tests/unit/proxy/test_get_image.py
tests/unit/proxy/test_prompt_test_endpoint.py
tests/unit/proxy/test_reducto_ocr_route.py
tests/unit/proxy/test_response_polling_pre_call_checks.py
tests/unit/proxy/test_ui_path_detection.py
workers: 4
dist: loadscope
timeout: 15
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}
workers: ${{ matrix.workers }}
reruns: 2
timeout-minutes: ${{ matrix.timeout }}
dist: ${{ matrix.dist }}
artifact-name: proxy-db-${{ matrix.test-group }}

223
.github/workflows/test-unit.yml vendored
View file

@ -28,11 +28,6 @@ concurrency:
# defaults. An absent matrix key renders as an empty string, which is not a
# number, so a partially-specified entry would fail the call rather than fall
# back to the default.
#
# tests/unit/proxy keeps its own caller (test-unit-proxy-db.yml): it is already
# a matrix and carries a shard-coverage guard that reads that file by name.
# Folding it in here is a follow-up, together with generalising that guard into
# assert_ci_coverage.py.
jobs:
rust-bridge:
name: Build the Rust bridge
@ -346,6 +341,16 @@ jobs:
timeout-minutes: 60
job-timeout-minutes: 100
- shard: mcp-elicitation
artifact-name: mcp-elicitation
test-path: >-
tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
workers: 2
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-infra
artifact-name: proxy-infra
test-path: >-
@ -498,3 +503,211 @@ jobs:
job-timeout-minutes: ${{ matrix.job-timeout-minutes }}
dist: ${{ matrix.dist || 'loadscope' }}
artifact-name: ${{ matrix.artifact-name }}
lens-python-310:
name: Lens Python 3.10
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
python-version: "3.10"
test-path: tests/unit/proxy/lens/test_inference.py
workers: 0
reruns: 0
timeout-minutes: 5
artifact-name: lens-python-310
# Fast guard — fails the workflow when a test directory or file inside a sharded
# tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
# so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list
# and reads the same test-path keys the coverage census does.
assert-shard-coverage:
runs-on: ubuntu-latest
timeout-minutes: 2
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Assert every test directory and file is claimed by a shard
run: python3 .github/scripts/assert_ci_coverage.py --shards
proxy-db:
needs: assert-shard-coverage
# Display only the semantic shard name in the checks UI instead of GHA's
# default "proxy-db (key-generation, tests/unit/proxy/…, 0, loadscope, 20)"
# which includes every matrix field and gets truncated past the test-path.
name: ${{ matrix.test-group }}
permissions:
contents: read
id-token: write
pull-requests: write
strategy:
fail-fast: false
matrix:
include:
# Must run serially — event-loop conflict with the logging worker.
- test-group: key-generation
test-path: >-
tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
workers: 0
dist: loadscope
timeout: 20
# ---- auth: split into 2 shards ----
- test-group: auth-checks
test-path: >-
tests/unit/proxy/auth/test_auth_checks.py
tests/unit/proxy/auth/test_user_api_key_auth.py
tests/unit/proxy/test_credential_slot_registry.py
tests/unit/proxy/test_deprecated_key_grace_period.py
workers: 4
dist: loadscope
timeout: 15
- test-group: jwt-and-keys
test-path: >-
tests/unit/proxy/auth/test_jwt.py
tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
tests/unit/proxy/test_proxy_custom_auth.py
workers: 4
dist: loadscope
timeout: 15
# ---- test_proxy_utils.py, single shard, worksteal distribution ----
- test-group: proxy-utils
test-path: >-
tests/unit/proxy/test_proxy_utils.py
workers: 4
dist: worksteal
timeout: 15
# ---- proxy server: split into 2 shards ----
- test-group: proxy-server-core
test-path: >-
tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py
tests/unit/proxy/test__lazy_features.py
tests/unit/proxy/test_aproxy_startup.py
tests/unit/proxy/test_proxy_server.py
workers: 4
dist: loadscope
timeout: 15
- test-group: proxy-runtime
test-path: >-
tests/unit/proxy/auth/test_multipart_bypass_repro.py
tests/unit/proxy/auth/test_proxy_routes.py
tests/unit/proxy/middleware/test_request_size_limit_middleware.py
tests/unit/proxy/test_proxy_config_unit_test.py
tests/unit/proxy/test_proxy_token_counter.py
tests/unit/proxy/test_server_root_path.py
workers: 4
dist: loadscope
timeout: 15
- test-group: mcp-oauth
test-path: >-
tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
tests/unit/proxy/_experimental/mcp_server/outbound_credentials
workers: 4
dist: loadscope
timeout: 15
# ---- logging: split into 2 shards ----
- test-group: custom-logging
test-path: >-
tests/proxy_unit_tests/test_proxy_custom_logger.py
tests/unit/proxy/test_custom_callback_input.py
tests/unit/proxy/test_custom_logger_s3_gcs.py
workers: 4
dist: loadscope
timeout: 15
- test-group: logging-misc
test-path: >-
tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
tests/unit/proxy/spend_tracking/test_search_api_logging.py
tests/unit/proxy/test_proxy_reject_logging.py
workers: 4
dist: loadscope
timeout: 15
- test-group: db-and-spend
test-path: >-
tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
tests/unit/proxy/db/test_update_daily_tag_spend.py
tests/unit/proxy/test_db_schema_changes.py
tests/unit/proxy/test_prisma_client_backoff_retry.py
tests/unit/proxy/test_update_spend.py
tests/unit/skills/test_skills_db.py
workers: 4
dist: loadscope
timeout: 15
# ---- guardrails + budget + hooks: split into 2 ----
- test-group: guardrails-hooks
test-path: >-
tests/unit/proxy/hooks/test_banned_keyword_list.py
tests/unit/proxy/test_proxy_setting_guardrails.py
tests/unit/proxy/test_unit_test_proxy_hooks.py
workers: 4
dist: loadscope
timeout: 15
- test-group: budgets
test-path: >-
tests/unit/proxy/auth/test_default_end_user_budget_simple.py
tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
tests/unit/proxy/test_zero_cost_model_budget_bypass.py
workers: 4
dist: loadscope
timeout: 15
- test-group: endpoints-and-responses
test-path: >-
tests/proxy_unit_tests/test_proxy_exception_mapping.py
tests/unit/proxy/lens
tests/unit/proxy/auth/test_models_fallback_endpoint.py
tests/unit/proxy/common_utils/test_check_batch_cost.py
tests/unit/proxy/common_utils/test_check_responses_cost.py
tests/unit/proxy/common_utils/test_realtime_cache.py
tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
tests/unit/proxy/response_polling
tests/unit/proxy/test_custom_tokenizer_bug.py
tests/unit/proxy/test_get_favicon.py
tests/unit/proxy/test_get_image.py
tests/unit/proxy/test_prompt_test_endpoint.py
tests/unit/proxy/test_reducto_ocr_route.py
tests/unit/proxy/test_response_polling_pre_call_checks.py
tests/unit/proxy/test_ui_path_detection.py
workers: 4
dist: loadscope
timeout: 15
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}
workers: ${{ matrix.workers }}
reruns: 2
timeout-minutes: ${{ matrix.timeout }}
dist: ${{ matrix.dist }}
artifact-name: proxy-db-${{ matrix.test-group }}
unit-passed:
name: unit passed
needs: [rust-bridge, unit, lens-python-310, assert-shard-coverage, proxy-db]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 2
permissions: {}
steps:
- name: Require every unit job to succeed
env:
NEEDS: ${{ toJSON(needs) }}
run: |
jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS"
jq -e 'all(.[]; .result == "success")' <<< "$NEEDS" > /dev/null

9
AGENTS.md
View file

@ -54,13 +54,13 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
Python max line length is 120, not 88
Never edit or commit `ruff-strict-budget.json`, `type-discipline-budget.json`, `basedpyright-code-budget.json`, or `test-quality-budget.json` on a PR branch, and don't run `make lint-budget-update` there. A scheduled Devin automation lowers the limits on the default branch in its own PR by exactly what landed since the last ratchet, so concurrent PRs don't fight over the same `"limit"` lines. Keep the hosted automation's target in sync when the repository default changes. If your branch already carries a budget edit, drop it before opening the PR
The four lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/test_quality_gate.py`, `scripts/type_check_gate.py`) compare each rule's codebase count on your branch against the count at its merge-base with the default branch, and a rule may not grow. There are no budget files to edit or ratchet: when a gate fails, fix the violations the branch introduced or remove at least as many of that rule elsewhere in the tree. The one exception is `reportAny` / `reportExplicitAny`, which share a fixed codebase-wide cap in `ANY_CAPS` in `scripts/type_check_gate.py` because Any spreads past the lines you touch. A branch may add Anys while the total stays under that cap. Only lower `ANY_CAPS`, and only in its own PR to the default branch, never raise it on a feature branch
`make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone budget gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their cap in `ANY_CAPS`, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
If you get an LIT001 fail, refactor the code to follow functional programming best practices rather than introducing mutable sequences or sets. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`set` and appending to it over time. Ideally, `# mutable-ok` is never used; reach for it only as a last resort when an immutable rewrite is impossible, and always pair it with a real reason. Plain `dict` is allowed: most of the Python ecosystem takes and returns dicts, and converting to `MappingProxyType` at every boundary costs more than it protects. Never deep copy a value just to hand out an immutable view; a defensive copy of a self-referential or large object is worse than the mutation it guards against
@ -97,6 +97,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
- Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
- Qualify every TypedDict field with `ReadOnly[...]` (LIT012), which nests freely with `Required` / `NotRequired` / `Annotated` in any order. If making the key writable is truly unavoidable, suppress with `# writable-ok: <reason>`
- Comprehensions take at most one `for` clause and one `if` clause (LIT014); split stacked clauses into a helper generator, a named intermediate, or a plain loop. Suppress with `# comprehension-ok: <reason>` only when unavoidable
- Every pydantic model must be frozen (LIT015), set via `model_config = ConfigDict(frozen=True)`, a dict-literal `model_config`, an inner `class Config`, or the class keywords. Subclasses inherit it unless they override it. Replace in-place field writes with `model_copy(update=...)`. If making the model mutable is truly unavoidable, suppress with `# frozen-ok: <reason>` on the `class` line
- Use dependency injection
- Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
- Use tagged unions + match
@ -129,4 +130,4 @@ Before implementing:
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify
Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or lint budgets to satisfy a check
Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or raise `ANY_CAPS` to satisfy a check

39
Makefile
View file

@ -6,9 +6,8 @@
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
test-rust-extension rust-sqlx-prepare lens-dev \
info lint lint-inner lint-dev lint-checks format \
lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
lint-test-quality lint-test-quality-budget-update \
lint-basedpyright lint-e2e-basedpyright lint-type-discipline \
lint-ruff-strict lint-gate lint-test-quality \
install-dev install-proxy-dev install-test-deps install-hooks \
install-helm-unittest check-circular-imports check-import-safety check check-inner pre-commit \
lint-install lint-fetch-base bootstrap
@ -32,13 +31,10 @@ help:
@echo " make lint-ruff - Run Ruff linting only"
@echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts"
@echo " make lint-e2e-basedpyright - Run basedpyright over tests/e2e and tests/e2e_harness (zero errors allowed)"
@echo " make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed"
@echo " make lint-format - Check ruff format formatting (matches CI)"
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
@echo " make lint-ruff-strict - Gate each strict ruff rule's codebase total against its merge-base count"
@echo " make lint-gate - Strict ruff gate in CI-parity mode (fetches the default branch, simulates the merge)"
@echo " make lint-ruff-budget-update - Ratchet ruff-strict-budget.json limits down by what this branch fixed"
@echo " make lint-test-quality - Gate the test suite against test-quality-budget.json"
@echo " make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + test quality + basedpyright)"
@echo " make lint-test-quality - Gate the test suite's TQ counts against their merge-base counts"
@echo " make check-circular-imports - Check for circular imports"
@echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
@ -143,7 +139,7 @@ lint-fetch-base:
# Mirror test-linting.yml's lint job environment: the proxy-dev group plus a generated
# Prisma client, so `basedpyright tests/e2e` resolves the same modules CI does. The
# budget gate itself no longer measures here (scripts/type_check_gate.py provisions its
# basedpyright gate itself no longer measures here (scripts/type_check_gate.py provisions its
# own .venv-typecheck). --inexact tops up the venv instead of pruning the proxy extras
# gen:api and the running proxy need.
lint-install:
@ -213,25 +209,20 @@ lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
$(UV_RUN) basedpyright tests/e2e tests/e2e_harness
# Type-discipline budget (mutable collections / casts / type guards / kwargs /
# Type-discipline gate (mutable collections / casts / type guards / kwargs /
# unexplained suppressions), the test-linting.yml step `make lint` used to omit.
lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/type_discipline_gate.py --base "$(BASE_REF)"
# Test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
# Test-quality gate (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
# litellm module-global mutation, credential-gated skips, conftest snapshot
# inventory), counted across tests/ the same delta-vs-base way.
lint-test-quality: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/test_quality_gate.py --base "$(BASE_REF)"
# --update lowers each limit by what this branch fixed since its branch point, so
# it needs the base ref fetched to resolve the merge-base.
lint-basedpyright-budget-update: install-dev
$(UV_RUN) python scripts/type_check_gate.py --update --base "$(BASE_REF)"
lint-format: format-check
lint-ruff-budget: install-dev
lint-ruff-strict: install-dev
$(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)"
# Strict gate, invoked the same way CI does in test-linting.yml so a local pass
@ -239,18 +230,6 @@ lint-ruff-budget: install-dev
lint-gate: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)"
lint-ruff-budget-update: install-dev
$(UV_RUN) python scripts/ruff_strict_gate.py --update --base "$(BASE_REF)"
lint-type-discipline-budget-update: install-dev
$(UV_RUN) python scripts/type_discipline_gate.py --update --base "$(BASE_REF)"
lint-test-quality-budget-update: install-dev
$(UV_RUN) python scripts/test_quality_gate.py --update --base "$(BASE_REF)"
# Ratchet all budgets in one shot (ruff strict + type-discipline + test quality + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-test-quality-budget-update lint-basedpyright-budget-update
check-circular-imports: $(LINT_DEP_INSTALL)
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
@ -260,7 +239,7 @@ check-import-safety: $(LINT_DEP_INSTALL)
# Combined linting, isomorphic to test-linting.yml's lint job so a local pass means a
# green CI lint: it installs the same env (proxy-dev + generated Prisma client) and then
# runs the diff-scoped ruff format check, whole-tree ruff check, the strict-rule /
# type-discipline / basedpyright budgets as a delta vs the base, then the circular-import
# type-discipline / basedpyright gates as a delta vs the base, then the circular-import
# and import-safety checks. Steps that compare against the base resolve it the same way CI
# does (merge-base with origin's current default branch). Setup (env sync, Prisma client,
# base fetch) runs once up front; the checks themselves are independent, so a sub-make

20
README.md
View file

@ -90,6 +90,24 @@ Managing LLM calls across providers gets complicated fast — different SDKs, au
uv add litellm
```
An independent `litellm-core` distribution provides the Python SDK with the same
`import litellm` API and runtime dependencies. It has no optional extras, CLI entry
points, or bundled dashboard. Install one SDK distribution per environment because
`litellm` and `litellm-core` own overlapping Python files. Use `litellm` for the
proxy, CLI, and optional extras
Build and install core from this checkout while its release integration is pending:
```shell
python scripts/build_core_distribution.py --out-dir dist/core
python -m pip install dist/core/litellm_core-*.whl
```
The builder requires Git, `uv` and the Rust build toolchain. It reads the release version
from the root `pyproject.toml`, leaves source files unchanged, and produces a wheel
and a self-contained sdist in `dist/core`. Run installation in a fresh environment
without `litellm`
```python
from litellm import completion
import os
@ -401,7 +419,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
| [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |
| [Volcengine (`volcengine`)](https://docs.litellm.ai/docs/providers/volcano) | ✅ | ✅ | ✅ | | | | | | | |
| [Voyage AI (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | |
| [VoyageAI by MongoDB (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | |
| [WandB Inference (`wandb`)](https://docs.litellm.ai/docs/providers/wandb_inference) | ✅ | ✅ | ✅ | | | | | | | |
| [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | |
| [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | |

146
basedpyright-code-budget.json
View file

@ -1,146 +0,0 @@
{
"reportAny": {
"limit": 13429
},
"reportArgumentType": {
"limit": 2198
},
"reportAssignmentType": {
"limit": 319
},
"reportAttributeAccessIssue": {
"limit": 480
},
"reportCallIssue": {
"limit": 112
},
"reportConstantRedefinition": {
"limit": 40
},
"reportDeprecated": {
"limit": 209
},
"reportDuplicateImport": {
"limit": 19
},
"reportExplicitAny": {
"limit": 3369
},
"reportFunctionMemberAccess": {
"limit": 7
},
"reportGeneralTypeIssues": {
"limit": 101
},
"reportIncompatibleMethodOverride": {
"limit": 56
},
"reportIncompatibleVariableOverride": {
"limit": 8
},
"reportInconsistentOverload": {
"limit": 12
},
"reportIndexIssue": {
"limit": 24
},
"reportInvalidTypeForm": {
"limit": 30
},
"reportInvalidTypeVarUse": {
"limit": 1
},
"reportMatchNotExhaustive": {
"limit": 0
},
"reportMissingParameterType": {
"limit": 5570
},
"reportMissingTypeArgument": {
"limit": 15281
},
"reportMissingTypeStubs": {
"limit": 40
},
"reportOperatorIssue": {
"limit": 0
},
"reportOptionalCall": {
"limit": 0
},
"reportOptionalIterable": {
"limit": 0
},
"reportOptionalMemberAccess": {
"limit": 0
},
"reportOptionalOperand": {
"limit": 0
},
"reportOptionalSubscript": {
"limit": 0
},
"reportPossiblyUnboundVariable": {
"limit": 56
},
"reportPrivateUsage": {
"limit": 1804
},
"reportRedeclaration": {
"limit": 8
},
"reportReturnType": {
"limit": 180
},
"reportTypedDictNotRequiredAccess": {
"limit": 22
},
"reportUndefinedVariable": {
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44802
},
"reportUnknownLambdaType": {
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38269
},
"reportUnknownParameterType": {
"limit": 19584
},
"reportUnknownVariableType": {
"limit": 29814
},
"reportUnnecessaryCast": {
"limit": 110
},
"reportUnnecessaryComparison": {
"limit": 687
},
"reportUnnecessaryContains": {
"limit": 4
},
"reportUnnecessaryIsInstance": {
"limit": 816
},
"reportUntypedBaseClass": {
"limit": 0
},
"reportUntypedFunctionDecorator": {
"limit": 27
},
"reportUnusedClass": {
"limit": 21
},
"reportUnusedFunction": {
"limit": 136
},
"reportUnusedImport": {
"limit": 542
},
"reportUnusedVariable": {
"limit": 137
}
}

127
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json
View file

@ -6436,6 +6436,133 @@
],
"title": "litellm_zero_cost_requests rate",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 438
},
"id": 112,
"panels": [],
"title": "Project model rate limits",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Configured rate limit for the Project on the requested model in the current window, by rate_limit_type",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never",
"spanNulls": false
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 439
},
"id": 113,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (project_id, project_alias, requested_model, rate_limit_type) (litellm_project_model_rate_limit_allowed_metric)",
"legendFormat": "{{project_alias}} ({{project_id}}) / {{requested_model}} / {{rate_limit_type}}",
"range": true,
"refId": "A"
}
],
"title": "litellm_project_model_rate_limit_allowed_metric",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Requests or tokens the Project has consumed on the requested model in the current rate limit window, by rate_limit_type",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineWidth": 1,
"showPoints": "never",
"spanNulls": false
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 439
},
"id": 114,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "max by (project_id, project_alias, requested_model, rate_limit_type) (litellm_project_model_rate_limit_used_metric)",
"legendFormat": "{{project_alias}} ({{project_id}}) / {{requested_model}} / {{rate_limit_type}}",
"range": true,
"refId": "A"
}
],
"title": "litellm_project_model_rate_limit_used_metric",
"type": "timeseries"
}
],
"preload": false,

28
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -732,6 +732,7 @@ class CheckBatchCost:
another pod claimed it. Raises on results-fetch or cost-computation
failures so the caller can leave the job unprocessed and retry it on a
later poll.
"""
from litellm.batches.batch_utils import (
count_error_file_failed_requests,
@ -815,15 +816,14 @@ class CheckBatchCost:
custom_llm_provider=custom_llm_provider,
)
# CheckBatchCost bypasses async_post_call_success_hook, so convert raw
# output/error file IDs to managed base64 IDs before the DB write here.
managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
if managed_files_hook is not None:
from litellm.proxy.openai_files_endpoints.common_utils import ManagedBatchOutputFileWriter
managed_files_hook: Final = self.proxy_logging_obj.get_proxy_hook("managed_files")
if isinstance(managed_files_hook, ManagedBatchOutputFileWriter):
managed_file_writer: Final = managed_files_hook
from litellm.proxy._types import UserAPIKeyAuth
managed_file_model_name = self._get_managed_file_model_name(
job=job, deployment_info=deployment_info
)
managed_file_model_name = self._get_managed_file_model_name(job=job, deployment_info=deployment_info)
_minimal_auth = UserAPIKeyAuth(
user_id=job.created_by or "default-user-id",
team_id=getattr(job, "team_id", None),
@ -832,17 +832,19 @@ class CheckBatchCost:
_raw_file_id = cast(str | None, getattr(response, _file_attr, None))
if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
try:
_unified_file_id = managed_files_hook.get_unified_output_file_id(
_unified_file_id = managed_file_writer.get_unified_output_file_id(
output_file_id=_raw_file_id,
model_id=model_id,
model_name=managed_file_model_name,
)
await managed_files_hook.store_unified_file_id(
file_id=_unified_file_id,
file_object=None,
await managed_file_writer.store_batch_output_file(
unified_file_id=_unified_file_id,
provider_file_id=_raw_file_id,
model_id=model_id,
model_name=managed_file_model_name,
owner=_minimal_auth,
litellm_parent_otel_span=None,
model_mappings={model_id: _raw_file_id},
user_api_key_dict=_minimal_auth,
size_bytes=len(content_bytes) if _file_attr == "output_file_id" else None,
)
setattr(response, _file_attr, _unified_file_id)
verbose_proxy_logger.info(

57
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py
View file

@ -6,7 +6,7 @@ same route are non-inference and free.
"""
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Dict, Final, Optional, Protocol, cast
from typing import TYPE_CHECKING, Dict, Final, Protocol, cast
import litellm
from litellm._logging import verbose_proxy_logger
@ -47,6 +47,10 @@ def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_Manag
return ManagedObjectRepository(prisma_client).table
def _is_response_gone_at_provider(error: Exception, provider_response_id: str) -> bool:
return getattr(error, "status_code", None) == 404 and provider_response_id in str(error)
class CheckResponsesCost:
def __init__(
self,
@ -61,10 +65,15 @@ class CheckResponsesCost:
self.prisma_client: PrismaClient = prisma_client
self.llm_router: Router = llm_router
def _resolve_deployment(self, response_id: str) -> bool:
model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
return model_id is not None and self.llm_router.get_deployment(model_id=model_id) is not None
async def _get_response(
self,
response_id: str,
litellm_metadata: Dict[str, str],
via_router: bool,
) -> ResponsesAPIResponse:
"""Fetch the upstream response, using deployment credentials when available.
@ -75,8 +84,7 @@ class CheckResponsesCost:
sees provider env vars, so it fails for every deployment whose credentials
live in the config; the row then never leaves ``queued``.
"""
model_id: Optional[str] = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
if model_id is None or self.llm_router.get_deployment(model_id=model_id) is None:
if not via_router:
return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
router_response = await self.llm_router.aget_responses(
response_id=response_id, litellm_metadata=litellm_metadata
@ -140,6 +148,8 @@ class CheckResponsesCost:
- Cost is tracked by the get-responses call, billed because the poll is stamped
with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
- Mark responses in a terminal state as complete in the database
- Mark responses the provider no longer has (404 through a resolved
deployment) as stale_expired
"""
try:
await self._cleanup_stale_managed_objects()
@ -159,6 +169,7 @@ class CheckResponsesCost:
verbose_proxy_logger.debug(f"Found {len(jobs)} response jobs to check")
completed_jobs: Final[list[_ManagedObjectRow]] = []
expired_jobs: Final[list[_ManagedObjectRow]] = []
for job in jobs:
unified_object_id = job.unified_object_id
@ -171,31 +182,48 @@ class CheckResponsesCost:
# Get the stored response object to extract model information
stored_response = job.file_object
model_name = stored_response.get("model", None)
# Decrypt the response ID
responses_id_security, _, _ = ResponsesIDSecurity()._decrypt_response_id(unified_object_id)
# Prepare metadata with model information for cost tracking
litellm_metadata = {
"user_api_key_user_id": job.created_by or "default-user-id",
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
}
# Add model information if available
if model_name:
litellm_metadata["model"] = model_name
litellm_metadata["model_group"] = model_name # Use same value for model_group
via_router = self._resolve_deployment(responses_id_security)
except Exception as e:
verbose_proxy_logger.warning(
f"Skipping job {unified_object_id} due to error: {e}"
)
continue
provider_response_id = ResponsesAPIRequestUtils.decode_responses_api_response_id(
responses_id_security
).get("response_id", responses_id_security)
try:
response = await self._get_response(
response_id=responses_id_security,
litellm_metadata=litellm_metadata,
via_router=via_router,
)
verbose_proxy_logger.debug(
f"Response {unified_object_id} status: {response.status}, model: {model_name}"
)
except Exception as e:
if via_router and _is_response_gone_at_provider(e, provider_response_id):
verbose_proxy_logger.info(
f"Response {unified_object_id} no longer available at provider (404), marking stale_expired: {e}"
)
expired_jobs.append(job)
continue
verbose_proxy_logger.warning(
f"Skipping job {unified_object_id} due to error: {e}"
)
@ -210,6 +238,7 @@ class CheckResponsesCost:
# Mark completed jobs in the database
if len(completed_jobs) > 0:
await _managed_object_table(self.prisma_client).update_many(
# bounded-ok: at most MAX_OBJECTS_PER_POLL_CYCLE rows per cycle, the find_many take above
where={"id": {"in": [job.id for job in completed_jobs]}},
data={"status": "completed"},
)
@ -217,3 +246,13 @@ class CheckResponsesCost:
f"Marked {len(completed_jobs)} response jobs as completed"
)
if len(expired_jobs) > 0:
await _managed_object_table(self.prisma_client).update_many(
# bounded-ok: at most MAX_OBJECTS_PER_POLL_CYCLE rows per cycle, the find_many take above
where={"id": {"in": [job.id for job in expired_jobs]}},
data={"status": "stale_expired"},
)
verbose_proxy_logger.info(
f"Marked {len(expired_jobs)} response jobs as stale_expired"
)

457
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -1,9 +1,11 @@
# What is this?
## This hook is used to check for LiteLLM managed files in the request body, and replace them with model-specific file id
import asyncio
import base64
import json
from collections.abc import Iterator, Mapping, Sequence
import time
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -23,21 +25,25 @@ from uuid import NAMESPACE_URL, uuid5
import httpx
from fastapi import HTTPException
from pydantic import ValidationError
from typing_extensions import ReadOnly
from typing_extensions import ReadOnly, Unpack
import litellm
from litellm import Router, verbose_logger
from litellm._internal_context import with_service_target
from litellm._uuid import uuid
from litellm.caching.caching import DualCache
from litellm.constants import MAX_FILE_LIST_LIMIT
from litellm.constants import (
BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS,
BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
MAX_FILE_LIST_LIMIT,
)
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_file_metadata,
)
from openai import AsyncOpenAI
from openai import APIConnectionError, AsyncOpenAI
from openai.types.file_deleted import FileDeleted
from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
@ -55,7 +61,10 @@ from litellm.proxy._types import (
ProxyException,
UserAPIKeyAuth,
)
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
sanitize_for_log,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
BATCH_CREATE_HIDDEN_PARAM,
FILE_LIST_CONTINUATION_CHUNK_SIZE,
@ -77,6 +86,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
request_tags_from_metadata,
)
from litellm.repositories.managed_file_repository import ManagedFileRepository
from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue]
AllMessageValues,
AsyncCursorPage,
@ -96,6 +106,17 @@ from litellm.types.utils import (
SpecialEnums,
)
class _ManagedFileRetrieve(Protocol):
async def __call__(
self,
*,
file_id: str,
_litellm_internal_model_credentials: Mapping[str, object] | None = None,
**kwargs: Unpack[FileRetrieveCallOptions],
) -> OpenAIFileObject: ...
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from prisma.models import (
@ -146,6 +167,91 @@ def _parse_managed_file_object(raw_file_object: object, unified_file_id: str) ->
return None
def _batch_output_file_object(
unified_file_id: str, raw_file_id: str, size_bytes: int, *, fallback: bool
) -> OpenAIFileObject:
filename: Final = raw_file_id.rsplit("/", 1)[-1] or raw_file_id
return OpenAIFileObject(
id=unified_file_id,
object="file",
purpose="batch_output",
filename=filename,
created_at=int(time.time()),
bytes=size_bytes,
status="processed",
litellm_details_fallback=True if fallback else None,
)
def _public_file_object(file_object: OpenAIFileObject, unified_file_id: str) -> OpenAIFileObject:
return file_object.model_copy(update={"id": unified_file_id, "litellm_details_fallback": None})
def _is_transient_file_retrieve_error(error: Exception) -> bool:
status_code: Final = getattr(error, "status_code", None)
if isinstance(status_code, int):
return status_code in {408, 429} or status_code >= 500
return isinstance(error, (httpx.TransportError, APIConnectionError, asyncio.TimeoutError))
def _proxy_llm_router() -> Router | None:
import litellm.proxy.proxy_server as proxy_server_module
return cast(Router | None, getattr(proxy_server_module, "llm_router", None))
def _provider_file_retrieve_credentials(
*,
llm_router: Router | None,
model_id: str | None,
) -> Mapping[str, object] | None:
if llm_router is None or model_id is None:
return None
try:
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id)
except Exception as error:
verbose_logger.warning(
"Failed to retrieve credentials for provider file "
f"model_id={sanitize_for_log(model_id)}: {sanitize_for_log(error)}"
)
return None
return cast(Mapping[str, object], credentials) if credentials else None
_PROVIDER_FILE_RETRIEVE_PROVIDERS: Final[frozenset[str]] = frozenset(
{
"openai",
"azure",
"gemini",
"vertex_ai",
"bedrock",
"hosted_vllm",
"litellm_proxy",
"manus",
"anthropic",
"mistral",
"xai",
}
)
def _model_name_file_retrieve_provider(model_name: str | None) -> FileRetrieveProvider | None:
if model_name is None:
return None
provider, separator, _ = model_name.partition("/")
if not separator or provider not in _PROVIDER_FILE_RETRIEVE_PROVIDERS:
return None
return cast(FileRetrieveProvider, provider)
def _has_provider_file_retrieve_route(
*,
router_credentials: Mapping[str, object] | None,
model_name: str | None,
) -> bool:
return bool(router_credentials) or _model_name_file_retrieve_provider(model_name) is not None
class _ManagedFileRow(Protocol):
unified_file_id: str
file_object: OpenAIFileObject
@ -161,6 +267,8 @@ class _ManagedFileRow(Protocol):
class _ManagedFileTableActions(Protocol):
async def find_first(self, where: Mapping[str, object]) -> Optional[_ManagedFileRow]: ...
async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
async def find_many(
self,
where: Mapping[str, object],
@ -253,13 +361,21 @@ def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, s
_MANAGED_FILES_TARGET: Final = "managed_files"
_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Class variables or attributes
def __init__(self, internal_usage_cache: InternalUsageCache, prisma_client: PrismaClient):
def __init__(
self,
internal_usage_cache: InternalUsageCache,
prisma_client: PrismaClient,
*,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
):
self.internal_usage_cache = internal_usage_cache
self.prisma_client = prisma_client
self._sleep = sleep
@staticmethod
def _get_prometheus_logger():
@ -595,14 +711,231 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
for file_id in provider_file_ids:
model_name = decode_model_from_file_id(file_id)
raw_file_id = get_original_file_id(file_id)
await self.store_unified_file_id(
file_id=file_id,
file_object=None,
await self.store_batch_output_file(
unified_file_id=file_id,
provider_file_id=raw_file_id,
model_id=model_name or None,
model_name=model_name,
litellm_parent_otel_span=litellm_parent_otel_span,
model_mappings={model_name: raw_file_id} if model_name else {},
user_api_key_dict=owner_identity,
owner=owner_identity,
)
async def _afile_retrieve_with_retries(
self,
*,
provider_file_id: str,
call_options: Mapping[str, object],
internal_model_credentials: Mapping[str, object] | None = None,
) -> OpenAIFileObject:
"""Retrieve provider file details with transient retries and SDK retries disabled."""
retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve)
retrieve_options: Final = cast(
FileRetrieveCallOptions,
{**call_options, "max_retries": 0},
)
for attempt in range(len(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS) + 1):
if attempt > 0:
await self._sleep(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS[attempt - 1])
try:
if internal_model_credentials is None:
return await retrieve_file(
file_id=provider_file_id,
**retrieve_options,
)
return await retrieve_file(
file_id=provider_file_id,
_litellm_internal_model_credentials=MappingProxyType(dict(internal_model_credentials)),
**retrieve_options,
)
except Exception as error:
if not _is_transient_file_retrieve_error(error) or attempt == len(
_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS
):
raise
raise RuntimeError("Provider file retrieve retry loop ended without a result")
async def _fetch_provider_file_object(
self,
*,
unified_file_id: str,
provider_file_id: str,
model_id: str | None,
model_name: str | None,
llm_router: Router | None = None,
raise_on_failure: bool = False,
allow_default_provider: bool = False,
) -> tuple[OpenAIFileObject | None, bool]:
"""Fetch provider file details through the configured route under a total timeout."""
route_llm_router: Final = llm_router if llm_router is not None else _proxy_llm_router()
router_credentials: Final = _provider_file_retrieve_credentials(
llm_router=route_llm_router,
model_id=model_id,
)
model_name_provider: Final = _model_name_file_retrieve_provider(model_name)
fetch_route_available: Final = _has_provider_file_retrieve_route(
router_credentials=router_credentials,
model_name=model_name,
)
default_provider_route: Final = allow_default_provider and route_llm_router is not None
if not fetch_route_available and not default_provider_route:
return None, False
try:
if router_credentials is not None:
provider_file_object: Final = await asyncio.wait_for(
self._afile_retrieve_with_retries(
provider_file_id=provider_file_id,
call_options=router_credentials,
internal_model_credentials=router_credentials,
),
timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
)
return provider_file_object.model_copy(update={"id": unified_file_id}), True
if model_name_provider is not None:
provider_file_object_by_model_name: Final = await asyncio.wait_for(
self._afile_retrieve_with_retries(
provider_file_id=provider_file_id,
call_options={"custom_llm_provider": model_name_provider},
),
timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
)
return (
provider_file_object_by_model_name.model_copy(update={"id": unified_file_id}),
True,
)
if default_provider_route:
provider_file_object_by_default_route: Final = await asyncio.wait_for(
self._afile_retrieve_with_retries(
provider_file_id=provider_file_id,
call_options=router_credentials or {},
),
timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
)
return (
provider_file_object_by_default_route.model_copy(update={"id": unified_file_id}),
True,
)
return None, False
except Exception as error:
verbose_logger.warning(
"Failed to retrieve batch file object for "
f"provider_file_id={sanitize_for_log(provider_file_id)}: "
f"{type(error).__name__} {sanitize_for_log(error)}"
)
if raise_on_failure:
if isinstance(error, TimeoutError) and not str(error):
raise TimeoutError(
"Provider file retrieve timed out "
f"after {BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS} seconds"
) from error
raise
return None, True
async def _save_refreshed_file_object(
self,
stored: LiteLLM_ManagedFileTable,
file_object: OpenAIFileObject,
) -> None:
if not await ManagedFileRepository(self.prisma_client).update_file_object(stored.unified_file_id, file_object):
return
refreshed_row: Final = stored.model_copy(update={"file_object": file_object})
await self.internal_usage_cache.async_set_cache(
key=stored.unified_file_id,
value=refreshed_row.model_dump(),
litellm_parent_otel_span=None,
)
async def store_batch_output_file(
self,
*,
unified_file_id: str,
provider_file_id: str,
model_id: str | None,
model_name: str | None = None,
owner: UserAPIKeyAuth,
litellm_parent_otel_span: Span | None,
size_bytes: int | None = None,
fetch_provider_details: bool = True,
) -> None:
"""Register batch output or error file metadata, optionally fetching provider details."""
stored_file: Final = await self.get_unified_file_id(unified_file_id, litellm_parent_otel_span)
stored_object: Final = stored_file.file_object if stored_file is not None else None
if not fetch_provider_details:
if stored_file is not None:
return
router_credentials: Final = _provider_file_retrieve_credentials(
llm_router=_proxy_llm_router(),
model_id=model_id,
)
file_object_without_provider_details: Final = _batch_output_file_object(
unified_file_id,
provider_file_id,
size_bytes or 0,
fallback=_has_provider_file_retrieve_route(
router_credentials=router_credentials,
model_name=model_name,
),
)
await self.store_unified_file_id(
file_id=unified_file_id,
file_object=file_object_without_provider_details,
litellm_parent_otel_span=litellm_parent_otel_span,
model_mappings={model_id: provider_file_id} if model_id else {},
user_api_key_dict=owner,
)
return
if stored_object is not None and not stored_object.litellm_details_fallback:
return
fallback_written_recently: Final = (
stored_object is not None
and time.time() - stored_object.created_at < BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS
)
provider_fetch_result: Final = (
(None, True)
if fallback_written_recently
else await self._fetch_provider_file_object(
unified_file_id=unified_file_id,
provider_file_id=provider_file_id,
model_id=model_id,
model_name=model_name,
)
)
provider_object, fetch_route_available = provider_fetch_result
if (
stored_file is not None
and provider_object is None
and (size_bytes is None or (stored_object is not None and stored_object.bytes == size_bytes))
):
return
file_object: Final = (
provider_object
if provider_object is not None
else (
stored_object.model_copy(update={"bytes": size_bytes})
if stored_object is not None and size_bytes is not None
else _batch_output_file_object(
unified_file_id,
provider_file_id,
size_bytes or 0,
fallback=fetch_route_available,
)
)
)
if stored_file is not None:
await self._save_refreshed_file_object(stored_file, file_object)
return
await self.store_unified_file_id(
file_id=unified_file_id,
file_object=file_object,
litellm_parent_otel_span=litellm_parent_otel_span,
model_mappings={model_id: provider_file_id} if model_id else {},
user_api_key_dict=owner,
)
async def list_user_batches(
self,
user_api_key_dict: UserAPIKeyAuth,
@ -745,6 +1078,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
user_api_key_dict=user_api_key_dict,
db_batch_object=row,
unified_batch_id=_is_base64_encoded_unified_file_id(row.unified_object_id),
fetch_provider_details=False,
)
except Exception as e:
verbose_logger.warning(f"Failed to resolve managed file ids for batch {row.unified_object_id}: {e}")
@ -806,7 +1140,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
}
)
return [
parsed_file_object.model_copy(update={"id": row.unified_file_id})
_public_file_object(parsed_file_object, row.unified_file_id)
for row in file_ids
if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
]
@ -1474,42 +1808,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
setattr(response, file_attr, unified_file_id)
# Use llm_router credentials when available. Without credentials,
# Azure and other auth-required providers return 500/401.
file_object = None
try:
# Import module and use getattr for better testability with mocks
import litellm.proxy.proxy_server as proxy_server_module
_llm_router = getattr(proxy_server_module, "llm_router", None)
if _llm_router is not None and model_id:
_creds = _llm_router.get_deployment_credentials_with_provider(model_id) or {}
file_object = await litellm.afile_retrieve(
file_id=provider_file_id,
**cast(FileRetrieveCallOptions, _creds),
)
else:
file_object = await litellm.afile_retrieve(
custom_llm_provider=cast(
FileRetrieveProvider,
model_name.split("/")[0] if model_name and "/" in model_name else "openai",
),
file_id=provider_file_id,
)
verbose_logger.debug(
f"Successfully retrieved file object for {file_attr}={provider_file_id}"
)
except Exception as e:
verbose_logger.warning(
f"Failed to retrieve file object for {file_attr}={provider_file_id}: {str(e)}. Storing with None and will fetch on-demand."
)
await self.store_unified_file_id(
file_id=unified_file_id,
file_object=file_object,
await self.store_batch_output_file(
unified_file_id=unified_file_id,
provider_file_id=provider_file_id,
model_id=model_id,
model_name=resolved_model_name,
owner=user_api_key_dict,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_mappings={model_id: provider_file_id},
user_api_key_dict=user_api_key_dict,
)
request_metadata: Final = data.get("litellm_metadata")
await self.store_unified_object_id(
@ -1612,6 +1917,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
async def afile_retrieve(
self, file_id: str, litellm_parent_otel_span: Optional[Span], llm_router: Optional[Router] = None
) -> OpenAIFileObject:
"""Return public details for a managed file ID, refreshing a basic entry when possible."""
stored_file_object = await self.get_unified_file_id(file_id, litellm_parent_otel_span)
# Case 1 : This is not a managed file
@ -1621,30 +1927,57 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Case 2: Managed file and the file object exists in the database
# The stored file_object has the raw provider ID. Replace with the unified ID
# so callers see a consistent ID (matching Case 3 which does response.id = file_id).
if stored_file_object and stored_file_object.file_object:
# Use model_copy to ensure the ID update persists (Pydantic v2 compatibility)
response = stored_file_object.file_object.model_copy(update={"id": file_id})
return response
if stored_file_object.file_object is not None:
file_object: Final = stored_file_object.file_object
if file_object.litellm_details_fallback and stored_file_object.model_mappings:
try:
model_id, provider_file_id = next(iter(stored_file_object.model_mappings.items()))
refreshed_file_object, _ = await self._fetch_provider_file_object(
unified_file_id=file_id,
provider_file_id=provider_file_id,
model_id=model_id,
model_name=model_id,
llm_router=llm_router,
)
if refreshed_file_object is None:
return _public_file_object(file_object, file_id)
await self._save_refreshed_file_object(stored_file_object, refreshed_file_object)
return _public_file_object(refreshed_file_object, file_id)
except Exception as error:
verbose_logger.warning(
"Failed to refresh batch file object for "
f"file_id={sanitize_for_log(file_id)}: {sanitize_for_log(error)}"
)
return _public_file_object(file_object, file_id)
# Case 3: Managed file exists in the database but not the file object (for. e.g the batch task might not have run)
# So we fetch the file object from the provider. We deliberately do not store the result to avoid interfering with batch cost tracking code.
if not llm_router:
model_mapping: Final = next(iter(stored_file_object.model_mappings.items()), None)
if model_mapping is None:
raise Exception(
f"LiteLLM Managed File object with id={file_id} has no file_object "
f"and llm_router is required to fetch from provider"
f"LiteLLM Managed File object with id={file_id} has no file_object and no provider route to fetch it"
)
model_id, model_file_id = model_mapping
try:
model_id, model_file_id = next(iter(stored_file_object.model_mappings.items()))
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id) or {}
response = await litellm.afile_retrieve(
file_id=model_file_id,
**cast(FileRetrieveCallOptions, credentials),
response, fetch_route_available = await self._fetch_provider_file_object(
unified_file_id=file_id,
provider_file_id=model_file_id,
model_id=model_id,
model_name=model_id,
llm_router=llm_router,
raise_on_failure=True,
allow_default_provider=True,
)
response.id = file_id # Replace with unified ID
return response
except Exception as e:
raise Exception(f"Failed to retrieve file {file_id} from provider: {str(e)}") from e
if not fetch_route_available:
raise Exception(
f"LiteLLM Managed File object with id={file_id} has no file_object and no provider route to fetch it"
)
if response is None:
raise ValueError("Provider file details could not be retrieved")
return _public_file_object(response, file_id)
async def afile_list(
self,
@ -1707,7 +2040,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
**cursor_args,
)
matches.extend(
parsed_file_object.model_copy(update={"id": row.unified_file_id})
_public_file_object(parsed_file_object, row.unified_file_id)
for row in chunk
if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
and (purpose is None or parsed_file_object.purpose == purpose)

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.74"
version = "0.1.75"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.74"
version = "0.1.75"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

24
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -11,7 +11,7 @@ import time
from collections.abc import Callable
from dataclasses import dataclass, replace
from pathlib import Path
from typing import TYPE_CHECKING, Final, Optional, Union
from typing import TYPE_CHECKING, Final, Optional, Protocol, Union
from urllib.parse import unquote, urlsplit
from litellm_proxy_extras import prisma_toolchain
@ -35,6 +35,12 @@ if TYPE_CHECKING:
import psycopg.sql
class LensCheckConnect(Protocol):
def __call__(
self, conninfo: str, *, connect_timeout: int, autocommit: bool
) -> "psycopg.Connection[tuple[object, ...]]": ...
def str_to_bool(value: Optional[str]) -> bool:
if value is None:
return False
@ -243,11 +249,9 @@ def _redact_credentials(text: str) -> str:
passwords: Final = sorted(_configured_database_passwords(), key=len, reverse=True)
alternation: Final = "|".join(re.escape(password) for password in passwords)
password_pattern: Final = (
re.compile(rf"(?P<lead>:|password=)(?:{alternation})(?=@|&|$|[\s'\"\]),])", re.IGNORECASE)
if passwords
else None
re.compile(rf"(?<![A-Za-z0-9_])(?:{alternation})(?![A-Za-z0-9_])", re.IGNORECASE) if passwords else None
)
result: Final = password_pattern.sub(rf"\g<lead>{_REDACTED}", text) if password_pattern is not None else text
result: Final = password_pattern.sub(_REDACTED, text) if password_pattern is not None else text
return _secret_shape_redactor()(result)
@ -690,7 +694,7 @@ class ProxyExtrasDBManager:
)
@staticmethod
def raise_if_lens_rename_pending() -> None:
def raise_if_lens_rename_pending(connect: LensCheckConnect | None = None) -> None:
database_url: Final = os.environ.get("DATABASE_URL")
if not database_url:
return
@ -698,8 +702,9 @@ class ProxyExtrasDBManager:
import psycopg
except ImportError as exc:
raise RuntimeError("Install psycopg to verify Lens data safety before prisma db push.") from exc
open_connection: Final = connect if connect is not None else psycopg.connect
try:
with psycopg.connect(
with open_connection(
ProxyExtrasDBManager._strip_prisma_query_params(database_url), connect_timeout=10, autocommit=True
) as connection:
legacy: Final = connection.execute(
@ -710,7 +715,8 @@ class ProxyExtrasDBManager:
).fetchone()
except psycopg.Error as exc:
raise RuntimeError(
"Cannot verify Lens data safety; refusing prisma db push. Check database connectivity and psycopg installation."
"Cannot verify Lens data safety; refusing prisma db push. "
f"The database check failed: {_redact_credentials(str(exc)).strip()}"
) from exc
if legacy is not None:
raise RuntimeError(
@ -1152,7 +1158,7 @@ class ProxyExtrasDBManager:
if next_budget.attempts_left < budget.attempts_left:
time.sleep(random.randrange(5, 15))
budget = next_budget # rebind-ok: the loop carries the budget from one migrate deploy pass to the next
budget = next_budget
raise RuntimeError(
f"Database migration failed after {MAX_MIGRATE_DEPLOY_ATTEMPTS} "

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.106"
version = "0.4.107"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.106"
version = "0.4.107"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

4
litellm-rust/.agents/skills/rust-string-enums/SKILL.md
View file

@ -35,7 +35,7 @@ pub enum EventType {
}
```
Use `#[strum(serialize_all = "snake_case")]` or another supported case style when it exactly matches the contract. Use explicit variant spellings otherwise. With multiple accepted aliases, set `to_string` to the existing canonical output: Strum Display otherwise selects the longest `serialize` spelling
Spell each variant explicitly with `#[strum(serialize = "...")]`; do not use `serialize_all`. With multiple accepted aliases, set `to_string` to the existing canonical output: Strum Display otherwise selects the longest `serialize` spelling
Use only the derives the contract needs. A deserialize-only type should remain deserialize-only. Do not add an unknown variant to a closed enum, or derive Serde for a type that currently has no serialization contract
@ -43,7 +43,7 @@ Plain Serde derives with `rename` or `rename_all` remain appropriate for closed
## Preserve behavior during migration
Read the type, its callers, and existing tests before changing it. Preserve canonical output, accepted aliases, case sensitivity, whitespace handling, unknown values, malformed-input rejection, and existing public conversion APIs. Keep conversions needed by callers or compatibility even when Serde no longer uses them
Read the type, its callers, and existing tests before changing it. Preserve canonical output, accepted aliases, case sensitivity, whitespace handling, unknown values, malformed-input rejection, and existing public conversion contracts. Wrappers that only delegate to a Strum-derived trait are removed rather than kept: callers use `FromStr` and `From<Enum> for &'static str` directly. Keep conversions that add behavior needed by callers or compatibility even when Serde no longer uses them
Use the workspace dependencies and enable `serde_with.workspace = true` in a crate only when needed. Check the versions and enabled features in `Cargo.toml` and `Cargo.lock` rather than upgrading dependencies for this refactor

7
litellm-rust/AGENTS.md
View file

@ -4,6 +4,10 @@ For diagnostic tracing changes, follow [.agents/skills/rust-tracing/SKILL.md](.a
For string-valued enums and their Serde conversions, follow [.agents/skills/rust-string-enums/SKILL.md](.agents/skills/rust-string-enums/SKILL.md)
For fieldless enums, derive `strum::VariantArray` and use `VARIANTS` instead of a hand-listed `ALL` array; derive Strum string conversions instead of hand-written variant-to-string matches
Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `str::parse`) with no `as_str`/`parse` wrapper that only delegates, and spell each variant with explicit `#[strum(serialize = "...")]` instead of `serialize_all`
## Test placement
- Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;`
@ -17,6 +21,9 @@ For string-valued enums and their Serde conversions, follow [.agents/skills/rust
Use [`#[rstest]`](https://docs.rs/rstest/latest/rstest/attr.rstest.html) for new and updated tests and [`#[fixture]`](https://docs.rs/rstest/latest/rstest/attr.fixture.html) for reusable setup, injected through typed test arguments. Express input variations as named `#[case::name(...)]` cases instead of loops or duplicated tests so each failure identifies its case. Keep behavior assertions in the test body and fixtures focused on setup. Use the workspace `rstest` dependency
- Never loop over inputs (`for`, `.iter().for_each`, `.all`) inside a test body; give each input its own `#[case::name(...)]`, or use `#[values(...)]` for a cross product
- Exception: a test pinning a Rust table against a repo-owned data file (for example `include_str!` of a JSON config) may iterate that file's entries
## Error definitions
- A crate's errors live in `src/error.rs`, defined with `thiserror`, and re-exported from `lib.rs`

7
litellm-rust/Cargo.lock generated
View file

@ -3485,6 +3485,7 @@ dependencies = [
"rstest",
"serde",
"serde_json",
"strum",
"thiserror 2.0.19",
"tokio",
]
@ -3692,7 +3693,9 @@ dependencies = [
"litellm-auth-types",
"rstest",
"serde",
"serde_json",
"serde_yaml_ng",
"strum",
"tempfile",
"thiserror 2.0.19",
]
@ -3990,6 +3993,7 @@ dependencies = [
"rustls-native-certs",
"serde",
"serde_json",
"strum",
"tempfile",
"thiserror 2.0.19",
"tokio",
@ -4248,6 +4252,7 @@ dependencies = [
name = "litellm-llms-types"
version = "0.1.0"
dependencies = [
"indexmap 2.14.0",
"macro_rules_attribute",
"rstest",
"schemars 1.2.2",
@ -4538,6 +4543,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.9",
"strum",
"tar",
"target-lexicon",
"tempfile",
@ -6512,6 +6518,7 @@ checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
dependencies = [
"chrono",
"dyn-clone",
"indexmap 2.14.0",
"ref-cast",
"schemars_derive 1.2.2",
"serde",

1
litellm-rust/crates/cache/Cargo.toml vendored
View file

@ -8,6 +8,7 @@ repository.workspace = true
[dependencies]
serde.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
strum.workspace = true
thiserror.workspace = true
[dev-dependencies]

57
litellm-rust/crates/cache/src/cache_type.rs vendored
View file

@ -1,57 +1,44 @@
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, Hash)]
#[derive(
Clone,
Copy,
Debug,
Deserialize,
Serialize,
PartialEq,
Eq,
Hash,
strum::EnumString,
strum::IntoStaticStr,
strum::VariantArray,
)]
pub enum CacheType {
#[serde(rename = "local")]
#[strum(serialize = "local")]
Local,
#[serde(rename = "redis")]
#[strum(serialize = "redis")]
Redis,
#[serde(rename = "redis-semantic")]
#[strum(serialize = "redis-semantic")]
RedisSemantic,
#[serde(rename = "valkey-semantic")]
#[strum(serialize = "valkey-semantic")]
ValkeySemantic,
#[serde(rename = "s3")]
#[strum(serialize = "s3")]
S3,
#[serde(rename = "disk")]
#[strum(serialize = "disk")]
Disk,
#[serde(rename = "qdrant-semantic")]
#[strum(serialize = "qdrant-semantic")]
QdrantSemantic,
#[serde(rename = "azure-blob")]
#[strum(serialize = "azure-blob")]
AzureBlob,
#[serde(rename = "gcs")]
#[strum(serialize = "gcs")]
Gcs,
}
impl CacheType {
pub const ALL: [Self; 9] = [
Self::Local,
Self::Redis,
Self::RedisSemantic,
Self::ValkeySemantic,
Self::S3,
Self::Disk,
Self::QdrantSemantic,
Self::AzureBlob,
Self::Gcs,
];
pub const fn as_python_name(self) -> &'static str {
match self {
Self::Local => "local",
Self::Redis => "redis",
Self::RedisSemantic => "redis-semantic",
Self::ValkeySemantic => "valkey-semantic",
Self::S3 => "s3",
Self::Disk => "disk",
Self::QdrantSemantic => "qdrant-semantic",
Self::AzureBlob => "azure-blob",
Self::Gcs => "gcs",
}
}
pub fn from_python_name(value: &str) -> Option<Self> {
Self::ALL
.into_iter()
.find(|cache_type| cache_type.as_python_name() == value)
}
}

16
litellm-rust/crates/cache/tests/cache_type.rs vendored
View file

@ -1,5 +1,6 @@
use litellm_cache::CacheType;
use rstest::rstest;
use strum::VariantArray;
#[rstest]
#[case(CacheType::Local, "local")]
@ -15,16 +16,16 @@ fn every_python_cache_type_has_one_round_trip_identity(
#[case] cache_type: CacheType,
#[case] name: &str,
) {
assert_eq!(cache_type.as_python_name(), name);
assert_eq!(CacheType::from_python_name(name), Some(cache_type));
assert_eq!(<&'static str>::from(cache_type), name);
assert_eq!(name.parse::<CacheType>().ok(), Some(cache_type));
assert_eq!(
serde_json::to_value(cache_type).unwrap(),
serde_json::Value::from(name)
);
assert_eq!(
CacheType::ALL
CacheType::VARIANTS
.iter()
.filter(|candidate| candidate.as_python_name() == name)
.filter(|candidate| <&'static str>::from(**candidate) == name)
.count(),
1
);
@ -33,7 +34,10 @@ fn every_python_cache_type_has_one_round_trip_identity(
#[rstest]
fn python_cache_types_are_listed_in_python_order() {
assert_eq!(
CacheType::ALL.map(CacheType::as_python_name),
CacheType::VARIANTS
.iter()
.map(|cache_type| <&'static str>::from(*cache_type))
.collect::<Vec<_>>(),
[
"local",
"redis",
@ -52,5 +56,5 @@ fn python_cache_types_are_listed_in_python_order() {
#[case::unknown("memcached")]
#[case::case_sensitive("Redis")]
fn unknown_python_names_have_no_cache_type(#[case] name: &str) {
assert_eq!(CacheType::from_python_name(name), None);
assert!(name.parse::<CacheType>().is_err());
}

9
litellm-rust/crates/callbacks-legacy-python/src/adapter.rs
View file

@ -936,9 +936,6 @@ mod payload_tests {
/// The payload phases of `Logging` on top of `StubLogger`, with `pre_call` handing the
/// payload to the case's `on_pre_call`.
const PAYLOAD_LOGGER: &CStr = c"
class Request:
pass
class PayloadLogger(StubLogger):
def update_from_kwargs(self, **update):
self.update = update
@ -954,7 +951,7 @@ class PayloadLogger(StubLogger):
self.record('post_call', None)
self.post = (original_response, api_key, additional_args)
request = Request()
bound = {}
kwargs = {}
logger = PayloadLogger()
on_pre_call = lambda additional_args: None
@ -1225,10 +1222,10 @@ on_pre_call = lambda args: observed.append(
def check():
assert observed == [(True, True)], observed
")]
#[case::request_attribute_behind_an_omitted_keyword(c"
#[case::bound_value_behind_an_omitted_keyword(c"
document = {'type': 'document_url', 'document_url': 'data:application/pdf;base64,YWJj'}
pages = [0]
request.document = document
bound['document'] = document
kwargs = {'pages': pages}
observed = []
on_pre_call = lambda args: observed.append(

44
litellm-rust/crates/callbacks-legacy-python/src/call.rs
View file

@ -13,21 +13,21 @@ use pyo3::{
pub struct PublicCall {
args: Py<PyTuple>,
kwargs: Py<PyDict>,
request: Py<PyAny>,
bound: Py<PyDict>,
}
impl PublicCall {
/// Copies the keyword arguments once, so the legacy path's rewrites never reach the
/// caller's own dict while every value keeps its identity.
pub fn capture(
request: &Bound<'_, PyAny>,
bound: &Bound<'_, PyDict>,
args: &Bound<'_, PyTuple>,
kwargs: &Bound<'_, PyDict>,
) -> PyResult<Self> {
Ok(Self {
args: args.clone().unbind(),
kwargs: kwargs.copy()?.unbind(),
request: request.clone().unbind(),
bound: bound.clone().unbind(),
})
}
@ -55,31 +55,30 @@ impl PublicCall {
py: Python<'py>,
name: &str,
) -> PyResult<Option<Bound<'py, PyAny>>> {
lookup(self.kwargs.bind(py), self.request.bind(py), name)
lookup(self.kwargs.bind(py), self.bound.bind(py), name)
}
pub(crate) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.args)?;
visit.call(&self.kwargs)?;
visit.call(&self.request)
visit.call(&self.bound)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_support::{local, local_dict};
fn capture<'py>(py: Python<'py>, source: &std::ffi::CStr) -> (PublicCall, Bound<'py, PyDict>) {
let locals = PyDict::new(py);
py.run(source, Some(&locals), Some(&locals)).unwrap();
let request = locals.get_item("request").unwrap().unwrap();
let kwargs = locals
.get_item("kwargs")
.unwrap()
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let call = PublicCall::capture(&request, &PyTuple::empty(py), &kwargs).unwrap();
let call = PublicCall::capture(
&local_dict(&locals, "bound"),
&PyTuple::empty(py),
&local_dict(&locals, "kwargs"),
)
.unwrap();
(call, locals)
}
@ -91,25 +90,22 @@ mod tests {
py,
c"
pages = [0]
class Request:
pass
request = Request()
bound = {'pages': [1]}
kwargs = {'pages': pages}
",
);
let caller = locals
.get_item("kwargs")
.unwrap()
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let caller = local_dict(&locals, "kwargs");
call.kwargs()
.bind(py)
.set_item("litellm_call_id", "call")
.unwrap();
assert!(!caller.contains("litellm_call_id").unwrap());
let pages = locals.get_item("pages").unwrap().unwrap();
assert!(call.lookup(py, "pages").unwrap().unwrap().is(&pages));
assert!(
call.lookup(py, "pages")
.unwrap()
.unwrap()
.is(local(&locals, "pages"))
);
});
}
}

24
litellm-rust/crates/callbacks-legacy-python/src/test_support.rs
View file

@ -173,21 +173,23 @@ pub(crate) fn local<'py>(locals: &Bound<'py, PyDict>, name: &str) -> Bound<'py,
locals.get_item(name).unwrap().unwrap()
}
/// A legacy call over the namespace's `kwargs` (or none) and `request` (or `None`).
pub(crate) fn local_dict<'py>(locals: &Bound<'py, PyDict>, name: &str) -> Bound<'py, PyDict> {
local(locals, name).cast_into().unwrap()
}
/// A legacy call over the namespace's `kwargs` and `bound` dicts, each empty when absent.
pub(crate) fn legacy_call(
py: Python<'_>,
locals: &Bound<'_, PyDict>,
asynchronous: bool,
) -> LegacyLogging {
let request = locals
.get_item("request")
.unwrap()
.unwrap_or_else(|| py.None().into_bound(py));
let kwargs = locals
.get_item("kwargs")
.unwrap()
.map(|kwargs| kwargs.cast_into::<PyDict>().unwrap())
.unwrap_or_else(|| PyDict::new(py));
let call = PublicCall::capture(&request, &PyTuple::empty(py), &kwargs).unwrap();
let dict = |name: &str| {
locals
.get_item(name)
.unwrap()
.map(|value| value.cast_into::<PyDict>().unwrap())
.unwrap_or_else(|| PyDict::new(py))
};
let call = PublicCall::capture(&dict("bound"), &PyTuple::empty(py), &dict("kwargs")).unwrap();
LegacyLogging::new(py, crate::LoggingOperation::Ocr, call, asynchronous)
}

2
litellm-rust/crates/config/Cargo.toml
View file

@ -9,8 +9,10 @@ repository.workspace = true
litellm-auth-types.workspace = true
serde.workspace = true
serde_yaml_ng = "0.10.0"
strum.workspace = true
thiserror.workspace = true
[dev-dependencies]
rstest.workspace = true
serde_json.workspace = true
tempfile.workspace = true

48
litellm-rust/crates/config/src/mcp.rs
View file

@ -41,57 +41,43 @@ impl fmt::Debug for McpServer {
}
}
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, strum::IntoStaticStr)]
#[serde(rename_all = "snake_case")]
pub enum McpTransport {
#[default]
#[strum(serialize = "http")]
Http,
#[strum(serialize = "sse")]
Sse,
#[strum(serialize = "stdio")]
Stdio,
}
impl McpTransport {
pub fn as_str(self) -> &'static str {
match self {
Self::Http => "http",
Self::Sse => "sse",
Self::Stdio => "stdio",
}
}
}
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, strum::IntoStaticStr)]
#[serde(rename_all = "snake_case")]
pub enum McpAuth {
#[strum(serialize = "none")]
None,
#[strum(serialize = "api_key")]
ApiKey,
#[strum(serialize = "bearer_token")]
BearerToken,
#[strum(serialize = "basic")]
Basic,
#[strum(serialize = "authorization")]
Authorization,
#[strum(serialize = "token")]
Token,
#[strum(serialize = "oauth2")]
Oauth2,
#[strum(serialize = "aws_sigv4")]
AwsSigv4,
#[strum(serialize = "oauth2_token_exchange")]
Oauth2TokenExchange,
#[strum(serialize = "oauth2_id_jag")]
Oauth2IdJag,
#[strum(serialize = "true_passthrough")]
TruePassthrough,
#[strum(serialize = "oauth_delegate")]
OauthDelegate,
}
impl McpAuth {
pub fn as_str(self) -> &'static str {
match self {
Self::None => "none",
Self::ApiKey => "api_key",
Self::BearerToken => "bearer_token",
Self::Basic => "basic",
Self::Authorization => "authorization",
Self::Token => "token",
Self::Oauth2 => "oauth2",
Self::AwsSigv4 => "aws_sigv4",
Self::Oauth2TokenExchange => "oauth2_token_exchange",
Self::Oauth2IdJag => "oauth2_id_jag",
Self::TruePassthrough => "true_passthrough",
Self::OauthDelegate => "oauth_delegate",
}
}
}

39
litellm-rust/crates/config/tests/mcp.rs Normal file
View file

@ -0,0 +1,39 @@
use litellm_config::{McpAuth, McpTransport};
use rstest::rstest;
use serde_json::json;
#[rstest]
#[case::http(McpTransport::Http, "http")]
#[case::sse(McpTransport::Sse, "sse")]
#[case::stdio(McpTransport::Stdio, "stdio")]
fn mcp_transport_as_str_matches_the_serde_spelling(
#[case] transport: McpTransport,
#[case] name: &str,
) {
assert_eq!(<&'static str>::from(transport), name);
assert_eq!(
serde_json::from_value::<McpTransport>(json!(name)).unwrap(),
transport
);
}
#[rstest]
#[case::none(McpAuth::None, "none")]
#[case::api_key(McpAuth::ApiKey, "api_key")]
#[case::bearer_token(McpAuth::BearerToken, "bearer_token")]
#[case::basic(McpAuth::Basic, "basic")]
#[case::authorization(McpAuth::Authorization, "authorization")]
#[case::token(McpAuth::Token, "token")]
#[case::oauth2(McpAuth::Oauth2, "oauth2")]
#[case::aws_sigv4(McpAuth::AwsSigv4, "aws_sigv4")]
#[case::oauth2_token_exchange(McpAuth::Oauth2TokenExchange, "oauth2_token_exchange")]
#[case::oauth2_id_jag(McpAuth::Oauth2IdJag, "oauth2_id_jag")]
#[case::true_passthrough(McpAuth::TruePassthrough, "true_passthrough")]
#[case::oauth_delegate(McpAuth::OauthDelegate, "oauth_delegate")]
fn mcp_auth_as_str_matches_the_serde_spelling(#[case] auth: McpAuth, #[case] name: &str) {
assert_eq!(<&'static str>::from(auth), name);
assert_eq!(
serde_json::from_value::<McpAuth>(json!(name)).unwrap(),
auth
);
}

11
litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs
View file

@ -7,17 +7,26 @@ pub struct CustomLlmProvider<'a> {
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumString, IntoStaticStr)]
#[strum(serialize_all = "snake_case")]
pub enum LlmProviders {
#[strum(serialize = "anthropic")]
Anthropic,
#[strum(serialize = "aws_textract")]
AwsTextract,
#[strum(serialize = "azure_ai")]
AzureAi,
#[strum(serialize = "bedrock")]
Bedrock,
#[strum(serialize = "cohere")]
Cohere,
#[strum(serialize = "mistral")]
Mistral,
#[strum(serialize = "openai")]
Openai,
#[strum(serialize = "openai_like")]
OpenaiLike,
#[strum(serialize = "reducto")]
Reducto,
#[strum(serialize = "vertex_ai")]
VertexAi,
}

9
litellm-rust/crates/core-utils/src/prompt_templates/factory.rs
View file

@ -17,18 +17,13 @@ pub const EMPTY_TEXT_PLACEHOLDER: &str =
"[System: Empty message content sanitised to satisfy protocol]";
#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq)]
#[strum(serialize_all = "snake_case")]
pub enum TurnRole {
#[strum(serialize = "user")]
User,
#[strum(serialize = "assistant")]
Assistant,
}
impl TurnRole {
pub fn as_str(self) -> &'static str {
self.into()
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Turn {
pub role: TurnRole,

4
litellm-rust/crates/gateway-mcp/src/configured.rs
View file

@ -49,8 +49,8 @@ fn info(name: &str, config: &McpServer) -> ServerInfo {
let identity = format!(
"{name}|{}|{}|{}|{}",
config.url.as_ref().map_or("", SecretValue::expose),
config.transport.as_str(),
config.auth_type.map_or("", McpAuth::as_str),
<&'static str>::from(config.transport),
config.auth_type.map_or("", <&'static str>::from),
config.alias.as_deref().unwrap_or("")
);
ServerInfo {

182
litellm-rust/crates/host-python/src/argument.rs
View file

@ -2,78 +2,162 @@ use pyo3::{prelude::*, types::PyDict};
pub fn lookup<'py>(
kwargs: &Bound<'py, PyDict>,
request: &Bound<'py, PyAny>,
bound: &Bound<'py, PyDict>,
name: &str,
) -> PyResult<Option<Bound<'py, PyAny>>> {
if let Some(value) = kwargs.get_item(name)? {
return Ok(Some(value));
match kwargs.get_item(name)? {
Some(value) => Ok(Some(value)),
None => bound.get_item(name),
}
if let Ok(bound) = request.cast::<PyDict>() {
return bound.get_item(name);
}
request.getattr_opt(name)
}
pub fn present<'py>(
kwargs: &Bound<'py, PyDict>,
bound: &Bound<'py, PyDict>,
name: &str,
) -> PyResult<Option<Bound<'py, PyAny>>> {
Ok(lookup(kwargs, bound, name)?.filter(|value| !value.is_none()))
}
/// What `original_function(*args, **kwargs)` sees: the signature base with the keyword
/// dict laid over it, so a rewritten keyword wins and a deleted keyword falls back to the
/// signature default.
pub fn effective_py_args<'py>(
base: &Bound<'py, PyDict>,
kwargs: &Bound<'py, PyDict>,
) -> PyResult<Bound<'py, PyDict>> {
// Shallow copy, like the Python path: nested values stay shared with the caller.
let merged = base.copy()?;
merged.update(kwargs.as_mapping())?;
Ok(merged)
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[test]
fn lookup_prefers_the_keyword_even_when_none_and_falls_back_to_the_request() {
fn dicts<'py>(
py: Python<'py>,
kwargs: &str,
bound: &str,
) -> (Bound<'py, PyDict>, Bound<'py, PyDict>) {
let eval = |source: &str| {
py.eval(&std::ffi::CString::new(source).unwrap(), None, None)
.unwrap()
.cast_into::<PyDict>()
.unwrap()
};
(eval(kwargs), eval(bound))
}
#[rstest]
#[case::keyword_wins(
"{'api_key': 'keyword'}",
"{'api_key': 'bound'}",
Some(Some("keyword"))
)]
#[case::explicit_none_wins("{'api_key': None}", "{'api_key': 'bound'}", Some(None))]
#[case::bound_fallback("{}", "{'api_key': 'bound'}", Some(Some("bound")))]
#[case::missing("{}", "{}", None)]
fn lookup_prefers_the_keyword_and_falls_back_to_bound(
#[case] kwargs: &str,
#[case] bound: &str,
#[case] expected: Option<Option<&str>>,
) {
crate::initialize_python();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
key = object()
document = {'type': 'document_url'}
class Request:
api_key = 'from-request'
api_base = 'from-request'
document = document
request = Request()
kwargs = {'api_key': key, 'api_base': None}
",
Some(&locals),
Some(&locals),
)
.unwrap();
let item = |name: &str| locals.get_item(name).unwrap().unwrap();
let kwargs = item("kwargs").cast_into::<PyDict>().unwrap();
let request = item("request");
let find = |name: &str| lookup(&kwargs, &request, name).unwrap();
assert!(find("api_key").unwrap().is(item("key")));
assert!(find("api_base").unwrap().is_none());
assert!(find("document").unwrap().is(item("document")));
assert!(find("model").is_none());
let (kwargs, bound) = dicts(py, kwargs, bound);
let value = lookup(&kwargs, &bound, "api_key")
.unwrap()
.map(|value| value.extract::<Option<String>>().unwrap());
assert_eq!(value, expected.map(|value| value.map(str::to_owned)));
});
}
#[rstest::rstest]
#[case::prepared_value("{'api_key': 'replacement'}", Some("replacement"))]
#[case::explicit_none("{'api_key': None}", None)]
#[case::bound_fallback("{}", Some("original"))]
fn prepared_mapping_overrides_bound_values(
#[case] source: &str,
#[rstest]
#[case::explicit_none_hides_bound("{'api_key': None}", "{'api_key': 'bound'}", None)]
#[case::bound_none("{}", "{'api_key': None}", None)]
#[case::bound_value("{}", "{'api_key': 'bound'}", Some("bound"))]
fn present_treats_none_as_unset(
#[case] kwargs: &str,
#[case] bound: &str,
#[case] expected: Option<&str>,
) {
crate::initialize_python();
Python::attach(|py| {
let (kwargs, bound) = dicts(py, kwargs, bound);
let value = present(&kwargs, &bound, "api_key")
.unwrap()
.map(|value| value.extract::<String>().unwrap());
assert_eq!(value.as_deref(), expected);
});
}
#[rstest]
fn lookup_returns_the_callers_object() {
crate::initialize_python();
Python::attach(|py| {
let document = PyDict::new(py);
let bound = PyDict::new(py);
bound.set_item("api_key", "original").unwrap();
let source = std::ffi::CString::new(source).unwrap();
let prepared = py
.eval(&source, None, None)
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let value = lookup(&prepared, bound.as_any(), "api_key")
bound.set_item("document", &document).unwrap();
let found = lookup(&PyDict::new(py), &bound, "document")
.unwrap()
.unwrap();
assert!(found.is(&document));
});
}
fn dict<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyDict> {
py.eval(&std::ffi::CString::new(source).unwrap(), None, None)
.unwrap()
.cast_into::<PyDict>()
.unwrap()
}
#[rstest]
#[case::keyword_wins("{'api_key': 'base'}", "{'api_key': 'keyword'}", Some(Some("keyword")))]
#[case::explicit_none_wins("{'api_key': 'base'}", "{'api_key': None}", Some(None))]
#[case::base_default("{'api_key': 'base'}", "{}", Some(Some("base")))]
#[case::keyword_only("{}", "{'api_key': 'keyword'}", Some(Some("keyword")))]
#[case::missing("{}", "{}", None)]
fn effective_lays_the_keywords_over_the_base(
#[case] base: &str,
#[case] kwargs: &str,
#[case] expected: Option<Option<&str>>,
) {
crate::initialize_python();
Python::attach(|py| {
let merged = effective_py_args(&dict(py, base), &dict(py, kwargs)).unwrap();
let value = merged
.get_item("api_key")
.unwrap()
.map(|value| value.extract::<Option<String>>().unwrap());
assert_eq!(value, expected.map(|value| value.map(str::to_owned)));
});
}
#[rstest]
fn effective_leaves_both_inputs_untouched_and_keeps_object_identity() {
crate::initialize_python();
Python::attach(|py| {
let document = PyDict::new(py);
let base = dict(py, "{'model': 'base', 'pages': None}");
let kwargs = PyDict::new(py);
kwargs.set_item("document", &document).unwrap();
let merged = effective_py_args(&base, &kwargs).unwrap();
merged.set_item("model", "merged").unwrap();
assert_eq!(
value.extract::<Option<String>>().unwrap().as_deref(),
expected
base.get_item("model")
.unwrap()
.unwrap()
.extract::<String>()
.unwrap(),
"base"
);
assert!(!kwargs.contains("model").unwrap());
assert!(merged.get_item("document").unwrap().unwrap().is(&document));
});
}
}

2
litellm-rust/crates/host-python/src/lib.rs
View file

@ -19,7 +19,7 @@ mod owned;
mod runtime;
mod services;
pub use argument::lookup;
pub use argument::{effective_py_args, lookup, present};
pub use binding::PythonBinding;
pub use conversion_cache::{FromPythonCache, ToPythonCache};
pub use driver::{CallOptions, run_call};

1
litellm-rust/crates/http/Cargo.toml
View file

@ -21,6 +21,7 @@ rustls-native-certs.workspace = true
tokio-tungstenite.workspace = true
serde.workspace = true
serde_json.workspace = true
strum.workspace = true
thiserror.workspace = true
tokio.workspace = true
tracing.workspace = true

61
litellm-rust/crates/http/src/tls.rs
View file

@ -42,30 +42,28 @@ impl KeyExchangeGroup {
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, strum::EnumString)]
#[strum(
parse_err_ty = Unsupported,
parse_err_fn = unsupported_cipher_token
)]
pub enum Tls12CipherSuite {
#[strum(serialize = "ECDHE-ECDSA-AES128-GCM-SHA256")]
EcdheEcdsaAes128Gcm,
#[strum(serialize = "ECDHE-ECDSA-AES256-GCM-SHA384")]
EcdheEcdsaAes256Gcm,
#[strum(serialize = "ECDHE-ECDSA-CHACHA20-POLY1305")]
EcdheEcdsaChacha20,
#[strum(serialize = "ECDHE-RSA-AES128-GCM-SHA256")]
EcdheRsaAes128Gcm,
#[strum(serialize = "ECDHE-RSA-AES256-GCM-SHA384")]
EcdheRsaAes256Gcm,
#[strum(serialize = "ECDHE-RSA-CHACHA20-POLY1305")]
EcdheRsaChacha20,
}
impl FromStr for Tls12CipherSuite {
type Err = Unsupported;
fn from_str(name: &str) -> Result<Self, Self::Err> {
match name {
"ECDHE-ECDSA-AES128-GCM-SHA256" => Ok(Self::EcdheEcdsaAes128Gcm),
"ECDHE-ECDSA-AES256-GCM-SHA384" => Ok(Self::EcdheEcdsaAes256Gcm),
"ECDHE-ECDSA-CHACHA20-POLY1305" => Ok(Self::EcdheEcdsaChacha20),
"ECDHE-RSA-AES128-GCM-SHA256" => Ok(Self::EcdheRsaAes128Gcm),
"ECDHE-RSA-AES256-GCM-SHA384" => Ok(Self::EcdheRsaAes256Gcm),
"ECDHE-RSA-CHACHA20-POLY1305" => Ok(Self::EcdheRsaChacha20),
_ => Err(Unsupported::CipherToken(name.to_owned())),
}
}
fn unsupported_cipher_token(name: &str) -> Unsupported {
Unsupported::CipherToken(name.to_owned())
}
impl Tls12CipherSuite {
@ -367,6 +365,39 @@ mod tests {
);
}
#[rstest]
#[case::ecdhe_ecdsa_aes128(
"ECDHE-ECDSA-AES128-GCM-SHA256",
Tls12CipherSuite::EcdheEcdsaAes128Gcm
)]
#[case::ecdhe_ecdsa_aes256(
"ECDHE-ECDSA-AES256-GCM-SHA384",
Tls12CipherSuite::EcdheEcdsaAes256Gcm
)]
#[case::ecdhe_ecdsa_chacha20(
"ECDHE-ECDSA-CHACHA20-POLY1305",
Tls12CipherSuite::EcdheEcdsaChacha20
)]
#[case::ecdhe_rsa_aes128("ECDHE-RSA-AES128-GCM-SHA256", Tls12CipherSuite::EcdheRsaAes128Gcm)]
#[case::ecdhe_rsa_aes256("ECDHE-RSA-AES256-GCM-SHA384", Tls12CipherSuite::EcdheRsaAes256Gcm)]
#[case::ecdhe_rsa_chacha20("ECDHE-RSA-CHACHA20-POLY1305", Tls12CipherSuite::EcdheRsaChacha20)]
fn tls12_cipher_suite_parses_each_openssl_name(
#[case] name: &str,
#[case] expected: Tls12CipherSuite,
) {
assert_eq!(name.parse::<Tls12CipherSuite>().unwrap(), expected);
}
#[rstest]
#[case::unknown("AES128-SHA")]
#[case::case_sensitive("ecdhe-rsa-aes256-gcm-sha384")]
fn unsupported_cipher_token_keeps_the_verbatim_name(#[case] name: &str) {
assert_eq!(
name.parse::<Tls12CipherSuite>().unwrap_err(),
Unsupported::CipherToken(name.to_owned())
);
}
#[test]
fn named_suites_are_the_only_tls12_suites_offered_and_tls13_stays() {
let tls = ClientConfig::try_from(&config(HttpSettings {

106
litellm-rust/crates/lens/tests/receiver.rs
View file

@ -10,7 +10,10 @@ use rstest::rstest;
use serde_json::json;
use sha2::{Digest, Sha256};
use std::{
sync::{Arc, atomic::Ordering},
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
time::Duration,
};
use wiremock::{
@ -116,6 +119,107 @@ async fn agent_picker_query_preserves_scope_through_the_internal_read_route() {
assert_eq!(response.json::<serde_json::Value>().await.unwrap(), result);
}
#[rstest]
#[case::list_paid("list", None, 0.75)]
#[case::list_zero("list", None, 0.0)]
#[case::detail_paid("trace", None, 0.75)]
#[case::paged_zero("trace", Some(1), 0.0)]
#[tokio::test]
async fn internal_reads_refresh_delayed_gateway_amounts(
#[case] operation: &str,
#[case] page_size: Option<u32>,
#[case] cost: f64,
) {
let store = MockServer::start().await;
let start_ms = (unix_seconds() as i64 - 600) * 1000;
let available = Arc::new(AtomicBool::new(false));
Mock::given(body_string_contains("FROM agent_traces_by_key"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": [{
"trace_id": "trace", "trace_ref": "ref", "team_id": "team",
"api_key_hash": "key", "user_id": "owner", "name": "model call",
"service": "agent", "input_preview": "", "status": "STATUS_CODE_OK",
"start_ms": start_ms, "duration_ms": 1, "span_count": 1,
"agent_count": 0, "agent_invocations": 0, "llm_calls": 1,
"tool_calls": 0, "input_tokens": 1, "output_tokens": 1,
"models": ["test-model"], "error_count": 0, "request_ids": []
}]})))
.mount(&store)
.await;
Mock::given(body_string_contains("o.SpanId AS span_id"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": [{
"trace_id": "trace", "span_id": "span", "parent_span_id": "",
"name": "model call", "type": "llm", "agent": "",
"status": "STATUS_CODE_OK", "status_message": "", "error_truncated": 0,
"start_ns": start_ms * 1_000_000, "duration_ns": 1_000_000,
"service": "agent", "input_preview": "", "model": "test-model",
"input_tokens": 1, "output_tokens": 1, "litellm_request_id": "",
"call_keys": ["provider_response:response"], "call_evidence": "complete",
"team_id": "team", "api_key_hash": "key", "user_id": "owner"
}]})))
.expect(2)
.mount(&store)
.await;
let spend_available = available.clone();
Mock::given(body_string_contains("FROM spend_logs FINAL"))
.respond_with(move |_: &wiremock::Request| {
let rows = if spend_available.load(Ordering::Acquire) {
json!([{
"request_id": "request", "litellm_call_id": "",
"response_id": "response", "upstream_response_id": "",
"trace_id": "", "span_id": "", "team_id": "team",
"api_key": "key", "user": "owner", "spend": cost,
"start_ms": start_ms
}])
} else {
json!([])
};
ResponseTemplate::new(200).set_body_json(json!({"data": rows}))
})
.expect(2)
.mount(&store)
.await;
let server = serve(&store.uri(), true).await;
let scope = json!({"all_teams": 0, "user_id": "owner", "team_ids": []});
let (request, summary_path) = if operation == "list" {
(
json!({
"operation": operation, "scope": scope, "start_ms": start_ms,
"end_ms": start_ms + 1000, "cursor": null, "limit": 50
}),
"/data/0",
)
} else {
(
json!({
"operation": operation, "scope": scope, "trace_id": "trace",
"trace_ref": "ref", "cursor": null, "page_size": page_size
}),
"/summary",
)
};
let client = http_client().unwrap();
for expected in [None, Some(cost)] {
if expected.is_some() {
available.store(true, Ordering::Release);
tokio::time::sleep(litellm_traces_cache::LIVE_TTL + Duration::from_millis(200)).await;
}
let response = client
.post(format!("{}/internal/read", server.url))
.bearer_auth(SERVICE_TOKEN)
.json(&request)
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
let body = response.json::<serde_json::Value>().await.unwrap();
let summary = body.pointer(summary_path).unwrap();
assert_eq!(summary["spend"], json!(expected));
assert_eq!(summary["priced_calls"], u64::from(expected.is_some()));
assert_eq!(summary["llm_calls"], 1);
assert!(!body.to_string().contains("gateway_spend_pending"));
}
}
#[rstest]
#[tokio::test]
async fn feedback_summary_query_preserves_scope_through_the_internal_read_route() {

6
litellm-rust/crates/llms-types/AGENTS.md
View file

@ -67,3 +67,9 @@ The same ownership rule applies to Messages, Responses, Chat Completions, OCR, a
- Follow the workspace test-placement and `rstest` rules
- Test provider transformations, header policy, and stream execution in their owning crates
- Do not test import locations or Rust source structure as substitutes for behavior
# references
## json_schema.rs
- https://json-schema.org/draft/2020-12/json-schema-core

3
litellm-rust/crates/llms-types/Cargo.toml
View file

@ -6,9 +6,10 @@ license.workspace = true
repository.workspace = true
[features]
schema = ["dep:schemars"]
schema = ["dep:schemars", "schemars/indexmap2"]
[dependencies]
indexmap = { version = "2.14.0", features = ["serde"] }
macro_rules_attribute.workspace = true
schemars = { workspace = true, optional = true }
serde.workspace = true

36
litellm-rust/crates/llms-types/src/formats/AGENTS.md Normal file
View file

@ -0,0 +1,36 @@
# references
These links describe shared format fields and discriminators. The official API reference is authoritative, and SDK sources only clarify shapes the reference leaves implicit. These contracts are partial typed projections, not exhaustive upstream schemas
## chat_completions.rs and chat_completions/content.rs
- https://developers.openai.com/api/reference/resources/chat
SDK wire definitions:
- https://github.com/openai/openai-python/blob/main/src/openai/types/chat/chat_completion_content_part_param.py
## responses/output.rs and responses/streaming_websocket.rs
- https://developers.openai.com/api/reference/resources/responses
- https://developers.openai.com/api/reference/resources/responses/streaming-events
SDK wire definitions:
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response.py
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response_output_item.py
- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/mcp_tool_call_error.py
## batches.rs
- https://developers.openai.com/api/reference/resources/batches
## audio_transcription.rs
- https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions
## ocr.rs
- https://docs.mistral.ai/api/endpoint/ocr
Messages references live in `messages/AGENTS.md`. `LiteLLMOcrResponse` follows the Mistral OCR response shape, and `OcrBoundingBox` describes the corner coordinates providers copy into the normalized `bbox`. Normalized `tables` and `keyValuePairs` stay open because providers pass through different native shapes

2
litellm-rust/crates/llms-types/src/formats/audio_transcription.rs
View file

@ -1,6 +1,6 @@
use serde_json::Value;
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AudioTranscriptionResponseData {
pub text: String,
}

6
litellm-rust/crates/llms-types/src/formats/batches.rs
View file

@ -1,4 +1,4 @@
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum BatchStatus {
@ -7,7 +7,7 @@ pub enum BatchStatus {
Completed,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq)]
pub struct BatchRequestCounts {
pub total: u64,
@ -15,7 +15,7 @@ pub struct BatchRequestCounts {
pub failed: u64,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq)]
pub struct BatchResponse {
pub id: String,

75
litellm-rust/crates/llms-types/src/formats/chat_completions.rs
View file

@ -1,51 +1,41 @@
use serde_json::{Map, Value};
use strum::IntoStaticStr;
mod content;
pub use content::{
ChatContentPart, ChatFile, ChatInputAudio, ChatLogprobs, ChatMediaUrl, ChatMediaUrlParameters,
ChatTokenLogprob, ChatTopLogprob, ChatVideoMetadata, PromptCacheBreakpoint, PromptCacheMode,
};
/// Reasoning effort level accepted or applied by the model.
#[macro_rules_attribute::apply(wire_type)]
#[derive(Copy, Eq, IntoStaticStr)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, IntoStaticStr, strum::EnumString, strum::VariantArray)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum ReasoningEffort {
#[strum(serialize = "none")]
None,
#[strum(serialize = "minimal")]
Minimal,
#[strum(serialize = "low")]
Low,
#[strum(serialize = "medium")]
Medium,
#[strum(serialize = "high")]
High,
#[strum(serialize = "xhigh")]
Xhigh,
#[strum(serialize = "max")]
Max,
}
impl ReasoningEffort {
pub const ALL: [Self; 7] = [
Self::None,
Self::Minimal,
Self::Low,
Self::Medium,
Self::High,
Self::Xhigh,
Self::Max,
];
pub fn as_str(self) -> &'static str {
self.into()
}
pub fn parse(value: &str) -> Option<Self> {
Self::ALL
.into_iter()
.find(|effort| effort.as_str() == value)
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ChatMessageContent {
Text(String),
Parts(Vec<Value>),
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatMessage {
pub role: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -56,7 +46,7 @@ pub struct ChatMessage {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionToolCallFunctionChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
@ -65,7 +55,7 @@ pub struct ChatCompletionToolCallFunctionChunk {
pub provider_specific_fields: Option<Map<String, Value>>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionToolCallChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
@ -75,7 +65,7 @@ pub struct ChatCompletionToolCallChunk {
pub index: i64,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ChatCompletionThinkingBlock {
Thinking {
@ -96,7 +86,7 @@ pub enum ChatCompletionThinkingBlock {
/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python
/// path reports so cost tracking sees the same numbers on either path.
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct PromptTokensDetails {
pub cached_tokens: u64,
@ -104,7 +94,7 @@ pub struct PromptTokensDetails {
pub text_tokens: u64,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatCompletionsUsage {
pub prompt_tokens: u64,
@ -113,7 +103,7 @@ pub struct ChatCompletionsUsage {
pub prompt_tokens_details: PromptTokensDetails,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsChoiceMessage {
pub role: String,
// Whether an empty turn is `None` or `""` is the provider's choice, not a
@ -123,7 +113,7 @@ pub struct ChatCompletionsChoiceMessage {
pub content: Option<String>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsChoice {
pub index: u64,
pub message: ChatCompletionsChoiceMessage,
@ -135,7 +125,7 @@ pub struct ChatCompletionsChoice {
/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the
/// `ModelResponse` it already created, and echoing the provider's own id here
/// would change it. Pinned by `response_carries_no_id` in the Anthropic chat transformation tests.
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionsResponse {
pub created: u64,
pub model: String,
@ -143,7 +133,7 @@ pub struct ChatCompletionsResponse {
pub usage: ChatCompletionsUsage,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatCompletionDelta {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -162,7 +152,7 @@ pub struct ChatCompletionDelta {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionStreamingChoice {
pub index: u64,
pub delta: ChatCompletionDelta,
@ -172,7 +162,7 @@ pub struct ChatCompletionStreamingChoice {
pub logprobs: Option<Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatCompletionChunk {
pub id: String,
pub created: u64,
@ -189,6 +179,7 @@ pub struct ChatCompletionChunk {
#[cfg(test)]
mod tests {
use rstest::rstest;
use strum::VariantArray;
use super::*;
@ -207,10 +198,10 @@ mod tests {
) {
assert_eq!(
serde_json::to_value(effort).unwrap(),
Value::String(effort.as_str().to_string())
Value::String(<&'static str>::from(effort).to_string())
);
assert_eq!(ReasoningEffort::parse(effort.as_str()), Some(effort));
assert!(ReasoningEffort::ALL.contains(&effort));
assert_eq!(<&'static str>::from(effort).parse(), Ok(effort));
assert!(ReasoningEffort::VARIANTS.contains(&effort));
}
#[rstest]
@ -218,6 +209,6 @@ mod tests {
#[case::uppercase("HIGH")]
#[case::empty("")]
fn reasoning_effort_parse_rejects(#[case] value: &str) {
assert_eq!(ReasoningEffort::parse(value), None);
assert!(value.parse::<ReasoningEffort>().is_err());
}
}

155
litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs Normal file
View file

@ -0,0 +1,155 @@
use serde_json::{Map, Value};
use crate::formats::messages::{CacheControl, CitationsConfig, ContentSource};
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ChatContentPart {
Text {
text: String,
#[serde(skip_serializing_if = "Option::is_none")]
cache_control: Option<CacheControl>,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
ImageUrl {
image_url: ChatMediaUrl,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
VideoUrl {
video_url: ChatMediaUrl,
#[serde(flatten)]
extra: Map<String, Value>,
},
InputAudio {
input_audio: ChatInputAudio,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
File {
file: Box<ChatFile>,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Document {
source: Box<ContentSource>,
#[serde(skip_serializing_if = "Option::is_none")]
title: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
context: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
citations: Option<CitationsConfig>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Refusal {
refusal: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct PromptCacheBreakpoint {
pub mode: PromptCacheMode,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum PromptCacheMode {
Explicit,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ChatMediaUrl {
Url(String),
Parameters(Box<ChatMediaUrlParameters>),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatMediaUrlParameters {
pub url: String,
pub detail: Option<String>,
pub format: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatInputAudio {
pub data: String,
pub format: String,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatFile {
pub file_data: Option<String>,
pub file_id: Option<String>,
pub filename: Option<String>,
pub format: Option<String>,
pub detail: Option<String>,
pub video_metadata: Option<ChatVideoMetadata>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatVideoMetadata {
pub fps: Option<serde_json::Number>,
pub start_offset: Option<String>,
pub end_offset: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ChatLogprobs {
pub content: Option<Vec<ChatTokenLogprob>>,
pub refusal: Option<Vec<ChatTokenLogprob>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatTokenLogprob {
pub token: String,
pub logprob: serde_json::Number,
pub bytes: Option<Vec<u8>>,
pub top_logprobs: Option<Vec<ChatTopLogprob>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ChatTopLogprob {
pub token: String,
pub logprob: serde_json::Number,
pub bytes: Option<Vec<u8>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

14
litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md
View file

@ -3,3 +3,17 @@ This directory owns shared Messages API data contracts and serialization: reques
Adapter contracts and execution inputs such as `MessagesTransformContext` belong in `llms/src/base_llm/messages`. Provider rewriting and interpretation belong in `llms/src/<provider>/messages`. Call envelopes, live streams, and call orchestration belong in `inference-messages`
Represent web-search results, encrypted-content fields, and thinking configuration as data here. Decisions to flatten results, remove encrypted content, select thinking budgets, or require beta headers belong to provider transformations. Data-shape validation belongs here, while model capability checks and request adaptation do not
# references
These upstream contracts document fields and discriminators. They are documentation links, not saved golden snapshots. Beta-only blocks and tools (MCP, compaction, tool changes, advisor, fallback, browser state, and toolsets) follow the beta reference
## Messages bodies, content, tools, and streaming
- https://platform.claude.com/docs/en/api/http/messages/create
- https://platform.claude.com/docs/en/api/messages/create.md
- https://platform.claude.com/docs/en/api/beta/messages/create.md
- https://platform.claude.com/docs/en/build-with-claude/streaming.md
- https://platform.claude.com/docs/en/build-with-claude/context-editing
Anthropic-compatible hosts document their deviations in the `llms/src/<provider>/messages` guides. Copy a host reference into `../../providers/AGENTS.md` only when that host gets a typed extension in `providers`

712
litellm-rust/crates/llms-types/src/formats/messages/content.rs Normal file
View file

@ -0,0 +1,712 @@
use serde_json::{Map, Value};
use super::{CacheControl, MessagesToolParam};
use crate::json_schema::JsonSchema;
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ContentSource {
Base64 {
media_type: String,
data: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
Url {
url: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
File {
file_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
Text {
media_type: String,
data: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
Content {
content: BlockContent,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum BlockContent {
Text(String),
Blocks(Vec<MessagesContentPart>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolCaller {
Direct {
#[serde(flatten)]
extra: Map<String, Value>,
},
#[serde(rename = "code_execution_20250825")]
CodeExecution {
tool_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
#[serde(rename = "code_execution_20260120")]
CodeExecution20260120 {
tool_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct CitationsConfig {
pub enabled: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct PageCitation {
pub cited_text: String,
pub document_index: u64,
pub document_title: Option<String>,
pub start_page_number: u64,
pub end_page_number: u64,
pub file_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct CharCitation {
pub cited_text: String,
pub document_index: u64,
pub document_title: Option<String>,
pub start_char_index: u64,
pub end_char_index: u64,
pub file_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ContentBlockCitation {
pub cited_text: String,
pub document_index: u64,
pub document_title: Option<String>,
pub start_block_index: u64,
pub end_block_index: u64,
pub file_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchCitation {
pub cited_text: String,
pub url: String,
pub encrypted_index: String,
pub title: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct SearchResultCitation {
pub cited_text: String,
pub search_result_index: u64,
pub source: String,
pub title: Option<String>,
pub start_block_index: u64,
pub end_block_index: u64,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum Citation {
PageLocation(PageCitation),
CharLocation(CharCitation),
WebSearchResultLocation(WebSearchCitation),
ContentBlockLocation(ContentBlockCitation),
SearchResultLocation(SearchResultCitation),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MessagesContentPart {
Text(TextBlock),
Image(ImageBlock),
Document(Box<DocumentBlock>),
SearchResult(SearchResultBlock),
Thinking(ThinkingBlock),
RedactedThinking(RedactedThinkingBlock),
ToolUse(ToolUseBlock),
ToolResult(ToolResultBlock),
ToolReference(ToolReferenceBlock),
BrowserState(BrowserStateBlock),
ServerToolUse(ServerToolUseBlock),
WebSearchToolResult(ServerToolResultBlock<WebSearchToolResultContent>),
WebFetchToolResult(ServerToolResultBlock<WebFetchToolResultContent>),
CodeExecutionToolResult(ServerToolResultBlock<CodeExecutionToolResultContent>),
BashCodeExecutionToolResult(ServerToolResultBlock<BashCodeExecutionToolResultContent>),
TextEditorCodeExecutionToolResult(
ServerToolResultBlock<TextEditorCodeExecutionToolResultContent>,
),
ToolSearchToolResult(ServerToolResultBlock<ToolSearchToolResultContent>),
AdvisorToolResult(ServerToolResultBlock<AdvisorToolResultContent>),
ContainerUpload(ContainerUploadBlock),
McpToolUse(McpToolUseBlock),
McpToolResult(McpToolResultBlock),
McpToolListing(McpToolListingBlock),
Compaction(CompactionBlock),
ToolAddition(ToolChangeBlock),
ToolRemoval(ToolChangeBlock),
Fallback(FallbackBlock),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct TextBlock {
pub text: String,
pub citations: Option<Vec<Citation>>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ImageBlock {
pub source: ContentSource,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct DocumentBlock {
pub source: ContentSource,
pub title: Option<String>,
pub context: Option<String>,
pub citations: Option<CitationsConfig>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct SearchResultBlock {
pub source: String,
pub title: String,
pub content: Vec<MessagesContentPart>,
pub citations: Option<CitationsConfig>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ThinkingBlock {
pub thinking: String,
pub signature: String,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct RedactedThinkingBlock {
pub data: String,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolUseBlock {
pub id: String,
pub name: String,
pub input: Map<String, Value>,
pub caller: Option<ToolCaller>,
pub toolset_name: Option<String>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolResultBlock {
pub tool_use_id: String,
pub content: Option<BlockContent>,
pub is_error: Option<bool>,
pub toolset_name: Option<String>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolReferenceBlock {
pub tool_name: String,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct BrowserStateBlock {
pub tabs: Vec<BrowserTab>,
pub state_changes: Option<Vec<BrowserStateChange>>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct BrowserTab {
pub tab_id: String,
pub title: String,
pub url: String,
pub active: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum BrowserStateChange {
TabOpened {
tab_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
DownloadStarted {
download_id: String,
url: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
DownloadCompleted {
download_id: String,
url: String,
#[serde(skip_serializing_if = "Option::is_none")]
path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
size_bytes: Option<u64>,
#[serde(flatten)]
extra: Map<String, Value>,
},
DownloadFailed {
download_id: String,
url: String,
#[serde(skip_serializing_if = "Option::is_none")]
error: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ServerToolUseBlock {
pub id: String,
pub name: String,
pub input: Map<String, Value>,
pub caller: Option<ToolCaller>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ServerToolResultBlock<C> {
pub tool_use_id: String,
pub content: C,
pub caller: Option<ToolCaller>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ServerToolError {
pub error_code: String,
pub error_message: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum WebSearchToolResultContent {
Error(WebSearchResultError),
Results(Vec<WebSearchResult>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchResultError {
#[serde(rename = "type")]
pub error_type: WebSearchResultErrorType,
pub error_code: String,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum WebSearchResultErrorType {
WebSearchToolResultError,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchResult {
#[serde(rename = "type")]
pub result_type: WebSearchResultType,
pub url: String,
pub title: String,
pub encrypted_content: String,
pub page_age: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum WebSearchResultType {
WebSearchResult,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum WebFetchToolResultContent {
WebFetchToolResultError(ServerToolError),
WebFetchResult(WebFetchResult),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebFetchResult {
pub url: String,
pub content: WebFetchDocument,
pub retrieved_at: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum WebFetchDocument {
Document(Box<DocumentBlock>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum CodeExecutionToolResultContent {
CodeExecutionToolResultError(ServerToolError),
CodeExecutionResult(CodeExecutionResult),
EncryptedCodeExecutionResult(EncryptedCodeExecutionResult),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum BashCodeExecutionToolResultContent {
BashCodeExecutionToolResultError(ServerToolError),
BashCodeExecutionResult(CodeExecutionResult<BashCodeExecutionOutput>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct CodeExecutionResult<O = CodeExecutionOutput> {
pub stdout: String,
pub stderr: String,
pub return_code: i64,
pub content: Vec<O>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct EncryptedCodeExecutionResult {
pub encrypted_stdout: String,
pub stderr: String,
pub return_code: i64,
pub content: Vec<CodeExecutionOutput>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum CodeExecutionOutput {
CodeExecutionOutput {
file_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum BashCodeExecutionOutput {
BashCodeExecutionOutput {
file_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum TextEditorCodeExecutionToolResultContent {
TextEditorCodeExecutionToolResultError(ServerToolError),
TextEditorCodeExecutionViewResult(TextEditorViewResult),
TextEditorCodeExecutionCreateResult(TextEditorCreateResult),
TextEditorCodeExecutionStrReplaceResult(TextEditorStrReplaceResult),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct TextEditorViewResult {
pub content: String,
pub file_type: TextEditorFileType,
pub num_lines: Option<u64>,
pub start_line: Option<u64>,
pub total_lines: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum TextEditorFileType {
Text,
Image,
Pdf,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct TextEditorCreateResult {
pub is_file_update: bool,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct TextEditorStrReplaceResult {
pub lines: Option<Vec<String>>,
pub new_lines: Option<u64>,
pub new_start: Option<u64>,
pub old_lines: Option<u64>,
pub old_start: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolSearchToolResultContent {
ToolSearchToolResultError(ServerToolError),
ToolSearchToolSearchResult(ToolSearchResult),
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolSearchResult {
pub tool_references: Vec<ToolSearchReference>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolSearchReference {
ToolReference(ToolReferenceBlock),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum AdvisorToolResultContent {
AdvisorToolResultError(ServerToolError),
AdvisorResult {
text: String,
#[serde(skip_serializing_if = "Option::is_none")]
stop_reason: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
AdvisorRedactedResult {
encrypted_content: String,
#[serde(skip_serializing_if = "Option::is_none")]
stop_reason: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ContainerUploadBlock {
pub file_id: String,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpToolUseBlock {
pub id: String,
pub name: String,
pub server_name: String,
pub input: Map<String, Value>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpToolResultBlock {
pub tool_use_id: String,
pub content: Option<McpToolResultContent>,
pub is_error: Option<bool>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum McpToolResultContent {
Text(String),
Blocks(Vec<McpToolResultText>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum McpToolResultText {
Text(TextBlock),
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpToolListingBlock {
pub mcp_server_name: String,
pub tools: Vec<McpListedTool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpListedTool {
pub name: String,
pub description: Option<String>,
pub input_schema: JsonSchema,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct CompactionBlock {
pub content: Option<String>,
pub encrypted_content: Option<String>,
pub signature: Option<String>,
pub tool_changes: Option<Vec<ToolChange>>,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolChange {
ToolAddition(ToolChangeBlock),
ToolRemoval(ToolChangeBlock),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolChangeBlock {
pub tool: ToolChangeTarget,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolChangeTarget {
ToolReference {
name: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
McpToolReference {
server_name: String,
name: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
McpToolsetReference {
server_name: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
ToolDefinition {
definition: Box<MessagesToolParam>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct FallbackBlock {
pub from: FallbackModel,
pub to: FallbackModel,
pub trigger: Option<FallbackTrigger>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct FallbackModel {
pub model: String,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum FallbackTrigger {
Refusal {
#[serde(skip_serializing_if = "Option::is_none")]
category: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}

340
litellm-rust/crates/llms-types/src/formats/messages/metadata.rs Normal file
View file

@ -0,0 +1,340 @@
use serde_json::{Map, Value};
use crate::json_schema::JsonSchema;
use crate::recognized::Recognized;
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesMetadata {
pub user_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct OutputFormat {
#[serde(rename = "type")]
pub format_type: OutputFormatType,
pub schema: JsonSchema,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum OutputFormatType {
JsonSchema,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesCompaction {
#[serde(rename = "type")]
pub compaction_type: CompactionType,
pub instructions: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum CompactionType {
Summarize,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesContainer {
pub id: Option<String>,
pub expires_at: Option<String>,
pub skills: Option<Vec<ContainerSkill>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ContainerSkill {
#[serde(rename = "type")]
pub skill_type: SkillType,
pub skill_id: String,
pub version: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum SkillType {
Anthropic,
Custom,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpServer {
#[serde(rename = "type")]
pub server_type: McpServerType,
pub url: String,
pub name: String,
pub authorization_token: Option<String>,
pub tool_configuration: Option<McpToolConfiguration>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum McpServerType {
Url,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct McpToolConfiguration {
pub allowed_tools: Option<Vec<String>>,
pub enabled: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct StopDetails {
#[serde(rename = "type")]
pub detail_type: Recognized<StopDetailsType>,
pub category: Option<String>,
pub explanation: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum StopDetailsType {
Refusal,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ContextManagementResponse {
pub applied_edits: Option<Vec<AppliedEdit>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct AppliedEdit {
#[serde(rename = "type")]
pub edit_type: Option<String>,
pub cleared_input_tokens: Option<u64>,
pub cleared_tool_uses: Option<u64>,
pub cleared_thinking_turns: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct Safeguard {
#[serde(rename = "type")]
pub safeguard_type: String,
pub classifier_context: Option<Map<String, Value>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[derive(
Clone,
Debug,
PartialEq,
Eq,
strum::Display,
strum::EnumString,
strum::AsRefStr,
serde_with::DeserializeFromStr,
serde_with::SerializeDisplay,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]
pub enum MessageRole {
#[strum(serialize = "user")]
User,
#[strum(serialize = "assistant")]
Assistant,
#[strum(serialize = "system")]
System,
#[strum(default, transparent)]
Other(String),
}
impl MessageRole {
pub fn as_str(&self) -> &str {
self.as_ref()
}
}
impl From<String> for MessageRole {
fn from(value: String) -> Self {
value.parse().unwrap_or_else(|never| match never {})
}
}
impl From<MessageRole> for String {
fn from(value: MessageRole) -> Self {
value.to_string()
}
}
#[derive(
Clone,
Debug,
PartialEq,
Eq,
strum::Display,
strum::EnumString,
strum::AsRefStr,
serde_with::DeserializeFromStr,
serde_with::SerializeDisplay,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]
pub enum MessageType {
#[strum(serialize = "message")]
Message,
#[strum(default, transparent)]
Other(String),
}
impl MessageType {
pub fn as_str(&self) -> &str {
self.as_ref()
}
}
impl From<String> for MessageType {
fn from(value: String) -> Self {
value.parse().unwrap_or_else(|never| match never {})
}
}
impl From<MessageType> for String {
fn from(value: MessageType) -> Self {
value.to_string()
}
}
#[derive(
Clone,
Debug,
PartialEq,
Eq,
strum::Display,
strum::EnumString,
strum::AsRefStr,
serde_with::DeserializeFromStr,
serde_with::SerializeDisplay,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]
pub enum StopReason {
#[strum(serialize = "end_turn")]
EndTurn,
#[strum(serialize = "max_tokens")]
MaxTokens,
#[strum(serialize = "stop_sequence")]
StopSequence,
#[strum(serialize = "tool_use")]
ToolUse,
#[strum(serialize = "refusal")]
Refusal,
#[strum(serialize = "compaction")]
Compaction,
#[strum(serialize = "pause_turn")]
PauseTurn,
#[strum(serialize = "model_context_window_exceeded")]
ModelContextWindowExceeded,
#[strum(default, transparent)]
Other(String),
}
impl StopReason {
pub fn as_str(&self) -> &str {
self.as_ref()
}
}
impl From<String> for StopReason {
fn from(value: String) -> Self {
value.parse().unwrap_or_else(|never| match never {})
}
}
impl From<StopReason> for String {
fn from(value: StopReason) -> Self {
value.to_string()
}
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ContainerReference {
Id(String),
Parameters(Box<MessagesContainer>),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesDiagnosticsParam {
#[serde(
default,
skip_serializing_if = "Option::is_none",
with = "::serde_with::rust::double_option"
)]
#[cfg_attr(feature = "schema", schemars(with = "Option<String>"))]
pub previous_message_id: Option<Option<String>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesDiagnostics {
pub cache_miss_reason: Option<Recognized<CacheMissReason>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum CacheMissReason {
ModelChanged(CacheMissedTokens),
SystemChanged(CacheMissedTokens),
ToolsChanged(CacheMissedTokens),
MessagesChanged(CacheMissedTokens),
PreviousMessageNotFound {
#[serde(flatten)]
extra: Map<String, Value>,
},
Unavailable {
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct CacheMissedTokens {
pub cache_missed_input_tokens: u64,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

45
litellm-rust/crates/llms-types/src/formats/messages/mod.rs
View file

@ -1,11 +1,54 @@
mod content;
mod metadata;
mod request;
mod response;
pub mod streaming;
mod tools;
mod usage;
pub use content::{
AdvisorToolResultContent, BashCodeExecutionOutput, BashCodeExecutionToolResultContent,
BlockContent, BrowserStateBlock, BrowserStateChange, BrowserTab, CharCitation, Citation,
CitationsConfig, CodeExecutionOutput, CodeExecutionResult, CodeExecutionToolResultContent,
CompactionBlock, ContainerUploadBlock, ContentBlockCitation, ContentSource, DocumentBlock,
EncryptedCodeExecutionResult, FallbackBlock, FallbackModel, FallbackTrigger, ImageBlock,
McpListedTool, McpToolListingBlock, McpToolResultBlock, McpToolResultContent,
McpToolResultText, McpToolUseBlock, MessagesContentPart, PageCitation, RedactedThinkingBlock,
SearchResultBlock, SearchResultCitation, ServerToolError, ServerToolResultBlock,
ServerToolUseBlock, TextBlock, TextEditorCodeExecutionToolResultContent,
TextEditorCreateResult, TextEditorFileType, TextEditorStrReplaceResult, TextEditorViewResult,
ThinkingBlock, ToolCaller, ToolChange, ToolChangeBlock, ToolChangeTarget, ToolReferenceBlock,
ToolResultBlock, ToolSearchReference, ToolSearchResult, ToolSearchToolResultContent,
ToolUseBlock, WebFetchDocument, WebFetchResult, WebFetchToolResultContent, WebSearchCitation,
WebSearchResult, WebSearchResultError, WebSearchResultErrorType, WebSearchResultType,
WebSearchToolResultContent,
};
pub use metadata::{
AppliedEdit, CacheMissReason, CacheMissedTokens, CompactionType, ContainerReference,
ContainerSkill, ContextManagementResponse, McpServer, McpServerType, McpToolConfiguration,
MessageRole, MessageType, MessagesCompaction, MessagesContainer, MessagesDiagnostics,
MessagesDiagnosticsParam, MessagesMetadata, OutputFormat, OutputFormatType, Safeguard,
SkillType, StopDetails, StopDetailsType, StopReason,
};
pub use request::{
AdaptiveThinking, CacheControl, ContentBlock, ContentBlockType, ContextEdit, ContextManagement,
DisabledThinking, EffortLevel, EnabledThinking, Message, MessageContent,
ContextTrigger, DisabledThinking, EffortLevel, EnabledThinking, Message, MessageContent,
MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig, Speed, SystemPrompt,
ThinkingConfig, ThinkingDisplay,
};
pub use response::MessagesResponse;
pub use tools::{
AdvisorTool, AdvisorToolName, AllowedCaller, BashToolName, BrowserToolsetConfigs,
BuiltinMessagesTool, ClientTool, CodeExecutionToolName, ComputerTool, ComputerTool20251124,
ComputerToolName, ComputerToolsetConfigs, CustomTool, CustomToolType, McpToolset,
MemoryToolName, MessagesToolParam, ResponseInclusion, ServerTool, StrReplaceBasedEditToolName,
StrReplaceEditorName, TextEditorTool20250728, ToolChoice, ToolChoiceType, ToolResultUrlSource,
ToolSearchBm25ToolName, ToolSearchRegexToolName, Toolset, ToolsetToolConfig,
UrlSourceToolReference, UserInputUrlSource, UserLocationType, WebFetchTool,
WebFetchTool20260309, WebFetchTool20260318, WebFetchToolName, WebFetchUrlSources,
WebSearchTool, WebSearchTool20260318, WebSearchToolName, WebSearchUserLocation,
};
pub use usage::{
CacheCreationUsage, MessagesOutputTokensDetails, MessagesUsage, ServerToolUsage,
UsageIteration, UsageIterationType,
};

96
litellm-rust/crates/llms-types/src/formats/messages/request.rs
View file

@ -4,14 +4,14 @@ use strum::IntoStaticStr;
use crate::formats::chat_completions::ReasoningEffort;
use crate::recognized::Recognized;
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum SystemPrompt {
Text(String),
Blocks(Vec<ContentBlock>),
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum MessageContent {
Text(String),
@ -30,16 +30,24 @@ pub enum MessageContent {
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]
#[strum(serialize_all = "snake_case")]
pub enum ContentBlockType {
#[strum(serialize = "text")]
Text,
#[strum(serialize = "thinking")]
Thinking,
#[strum(serialize = "redacted_thinking")]
RedactedThinking,
#[strum(serialize = "tool_use")]
ToolUse,
#[strum(serialize = "server_tool_use")]
ServerToolUse,
#[strum(serialize = "tool_result")]
ToolResult,
#[strum(serialize = "compaction")]
Compaction,
#[strum(serialize = "advisor_tool_result")]
AdvisorToolResult,
#[strum(serialize = "web_search_tool_result")]
WebSearchToolResult,
#[strum(default, transparent)]
Other(String),
@ -57,7 +65,7 @@ impl From<ContentBlockType> for String {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ContentBlock {
#[serde(rename = "type", default, skip_serializing_if = "Option::is_none")]
@ -102,7 +110,7 @@ impl ContentBlock {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct CacheControl {
#[serde(rename = "type", skip_serializing_if = "Option::is_none")]
@ -115,7 +123,7 @@ pub struct CacheControl {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct Message {
pub role: String,
pub content: MessageContent,
@ -123,24 +131,22 @@ pub struct Message {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[derive(Copy, Hash, IntoStaticStr, Eq)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Hash, IntoStaticStr, Eq, strum::VariantArray)]
#[serde(rename_all = "lowercase")]
#[strum(serialize_all = "lowercase")]
pub enum EffortLevel {
#[strum(serialize = "low")]
Low,
#[strum(serialize = "medium")]
Medium,
#[strum(serialize = "high")]
High,
#[strum(serialize = "xhigh")]
Xhigh,
#[strum(serialize = "max")]
Max,
}
impl EffortLevel {
pub fn as_str(self) -> &'static str {
self.into()
}
}
impl From<EffortLevel> for ReasoningEffort {
fn from(level: EffortLevel) -> Self {
match level {
@ -153,24 +159,19 @@ impl From<EffortLevel> for ReasoningEffort {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, IntoStaticStr, Eq)]
#[serde(rename_all = "lowercase")]
#[strum(serialize_all = "lowercase")]
pub enum Speed {
#[strum(serialize = "fast")]
Fast,
#[strum(serialize = "standard")]
Standard,
}
impl Speed {
pub fn as_str(self) -> &'static str {
self.into()
}
}
/// The tools whose presence changes how the request is sent. Every other tool, custom or
/// server, deserializes as `Recognized::Unrecognized` and passes through verbatim.
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type")]
pub enum MessagesTool {
#[serde(rename = "advisor_20260301")]
@ -190,7 +191,22 @@ pub enum MessagesTool {
},
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ContextTrigger {
InputTokens {
value: u64,
#[serde(flatten)]
extra: Map<String, Value>,
},
ToolUses {
value: u64,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type")]
pub enum ContextEdit {
#[serde(rename = "compact_20260112")]
@ -210,7 +226,7 @@ pub enum ContextEdit {
},
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ContextManagement {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -219,7 +235,7 @@ pub struct ContextManagement {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OutputConfig {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -236,7 +252,7 @@ impl OutputConfig {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "lowercase")]
pub enum ThinkingDisplay {
@ -245,7 +261,7 @@ pub enum ThinkingDisplay {
Updates,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct EnabledThinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -256,7 +272,7 @@ pub struct EnabledThinking {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct AdaptiveThinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -265,14 +281,14 @@ pub struct AdaptiveThinking {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct DisabledThinking {
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "lowercase")]
pub enum ThinkingConfig {
Enabled(EnabledThinking),
@ -296,7 +312,7 @@ impl ThinkingConfig {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesRequest {
pub model: String,
pub messages: Vec<Message>,
@ -304,7 +320,7 @@ pub struct MessagesRequest {
pub params: MessagesOptionalParams,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesOptionalParams {
#[serde(skip_serializing_if = "Option::is_none")]
@ -676,7 +692,10 @@ mod tests {
#[rstest]
fn speed_names_match_the_wire(#[values(Speed::Fast, Speed::Standard)] speed: Speed) {
assert_eq!(serde_json::to_value(speed).unwrap(), json!(speed.as_str()));
assert_eq!(
serde_json::to_value(speed).unwrap(),
json!(<&'static str>::from(speed))
);
}
#[rstest]
@ -690,10 +709,13 @@ mod tests {
)]
level: EffortLevel,
) {
assert_eq!(serde_json::to_value(level).unwrap(), json!(level.as_str()));
assert_eq!(
serde_json::to_value(level).unwrap(),
json!(<&'static str>::from(level))
);
assert_eq!(
serde_json::to_value(ReasoningEffort::from(level)).unwrap(),
json!(level.as_str())
json!(<&'static str>::from(level))
);
}
}

2
litellm-rust/crates/llms-types/src/formats/messages/response.rs
View file

@ -1,6 +1,6 @@
use serde_json::{Map, Value};
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesResponse {
pub id: String,
#[serde(rename = "type")]

14
litellm-rust/crates/llms-types/src/formats/messages/streaming.rs
View file

@ -1,6 +1,6 @@
use serde_json::{Map, Value};
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesStreamUsage {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -17,7 +17,7 @@ pub struct MessagesStreamUsage {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesStreamMessage {
pub id: String,
#[serde(rename = "type")]
@ -32,7 +32,7 @@ pub struct MessagesStreamMessage {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MessagesContentBlockDelta {
TextDelta {
@ -56,7 +56,7 @@ pub enum MessagesContentBlockDelta {
},
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesContentBlock {
#[serde(rename = "type")]
pub block_type: String,
@ -82,7 +82,7 @@ pub struct MessagesContentBlock {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesDelta {
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -97,7 +97,7 @@ pub struct MessagesDelta {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MessagesStreamError {
#[serde(rename = "type")]
pub error_type: String,
@ -108,7 +108,7 @@ pub struct MessagesStreamError {
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MessagesStreamEvent {
MessageStart {

564
litellm-rust/crates/llms-types/src/formats/messages/tools.rs Normal file
View file

@ -0,0 +1,564 @@
use indexmap::IndexMap;
use serde_json::{Map, Value};
use super::{CacheControl, CitationsConfig, McpListedTool};
use crate::json_schema::JsonSchema;
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
pub enum AllowedCaller {
#[serde(rename = "direct")]
Direct,
#[serde(rename = "code_execution_20250825")]
CodeExecution20250825,
#[serde(rename = "code_execution_20260120")]
CodeExecution20260120,
#[serde(rename = "code_execution_20260521")]
CodeExecution20260521,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchUserLocation {
#[serde(rename = "type")]
pub location_type: UserLocationType,
pub city: Option<String>,
pub country: Option<String>,
pub region: Option<String>,
pub timezone: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum UserLocationType {
Approximate,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolChoice {
#[serde(rename = "type")]
pub choice_type: ToolChoiceType,
pub name: Option<String>,
pub disable_parallel_tool_use: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum ToolChoiceType {
Auto,
Any,
Tool,
None,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct CustomTool {
#[serde(rename = "type")]
pub tool_type: Option<CustomToolType>,
pub name: String,
pub input_schema: JsonSchema,
pub description: Option<String>,
pub strict: Option<bool>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub input_examples: Option<Vec<Map<String, Value>>>,
pub eager_input_streaming: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum CustomToolType {
Custom,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum BashToolName {
Bash,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum StrReplaceEditorName {
StrReplaceEditor,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum StrReplaceBasedEditToolName {
StrReplaceBasedEditTool,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum MemoryToolName {
Memory,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ComputerToolName {
Computer,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum CodeExecutionToolName {
CodeExecution,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ToolSearchRegexToolName {
ToolSearchToolRegex,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ToolSearchBm25ToolName {
ToolSearchToolBm25,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum WebSearchToolName {
WebSearch,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum WebFetchToolName {
WebFetch,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum AdvisorToolName {
Advisor,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ServerTool<N> {
pub name: N,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ClientTool<N> {
pub name: N,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub input_examples: Option<Vec<Map<String, Value>>>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct TextEditorTool20250728 {
pub name: StrReplaceBasedEditToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub input_examples: Option<Vec<Map<String, Value>>>,
pub max_characters: Option<u64>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ComputerTool {
pub name: ComputerToolName,
pub display_width_px: u64,
pub display_height_px: u64,
pub display_number: Option<u64>,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub input_examples: Option<Vec<Map<String, Value>>>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ComputerTool20251124 {
pub name: ComputerToolName,
pub display_width_px: u64,
pub display_height_px: u64,
pub display_number: Option<u64>,
pub enable_zoom: Option<bool>,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub input_examples: Option<Vec<Map<String, Value>>>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ResponseInclusion {
Full,
Excluded,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchTool {
pub name: WebSearchToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub allowed_domains: Option<Vec<String>>,
pub blocked_domains: Option<Vec<String>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub max_uses: Option<u64>,
pub strict: Option<bool>,
pub user_location: Option<WebSearchUserLocation>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebSearchTool20260318 {
pub name: WebSearchToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub allowed_domains: Option<Vec<String>>,
pub blocked_domains: Option<Vec<String>>,
pub cache_control: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub max_uses: Option<u64>,
pub response_inclusion: Option<ResponseInclusion>,
pub strict: Option<bool>,
pub user_location: Option<WebSearchUserLocation>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum UrlSourceToolReference {
ToolReference {
name: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ToolResultUrlSource {
All {
#[serde(flatten)]
extra: Map<String, Value>,
},
None {
#[serde(flatten)]
extra: Map<String, Value>,
},
Only {
tools: Vec<UrlSourceToolReference>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Except {
tools: Vec<UrlSourceToolReference>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum UserInputUrlSource {
All {
#[serde(flatten)]
extra: Map<String, Value>,
},
None {
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebFetchUrlSources {
pub client_tool_results: Option<ToolResultUrlSource>,
pub server_tool_results: Option<ToolResultUrlSource>,
pub user_input: Option<UserInputUrlSource>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebFetchTool {
pub name: WebFetchToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub allowed_domains: Option<Vec<String>>,
pub blocked_domains: Option<Vec<String>>,
pub cache_control: Option<CacheControl>,
pub citations: Option<CitationsConfig>,
pub defer_loading: Option<bool>,
pub max_content_tokens: Option<u64>,
pub max_uses: Option<u64>,
pub strict: Option<bool>,
pub url_sources: Option<WebFetchUrlSources>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebFetchTool20260309 {
pub name: WebFetchToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub allowed_domains: Option<Vec<String>>,
pub blocked_domains: Option<Vec<String>>,
pub cache_control: Option<CacheControl>,
pub citations: Option<CitationsConfig>,
pub defer_loading: Option<bool>,
pub max_content_tokens: Option<u64>,
pub max_uses: Option<u64>,
pub strict: Option<bool>,
pub url_sources: Option<WebFetchUrlSources>,
pub use_cache: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct WebFetchTool20260318 {
pub name: WebFetchToolName,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub allowed_domains: Option<Vec<String>>,
pub blocked_domains: Option<Vec<String>>,
pub cache_control: Option<CacheControl>,
pub citations: Option<CitationsConfig>,
pub defer_loading: Option<bool>,
pub max_content_tokens: Option<u64>,
pub max_uses: Option<u64>,
pub response_inclusion: Option<ResponseInclusion>,
pub strict: Option<bool>,
pub url_sources: Option<WebFetchUrlSources>,
pub use_cache: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AdvisorTool {
pub name: AdvisorToolName,
pub model: String,
pub allowed_callers: Option<Vec<AllowedCaller>>,
pub cache_control: Option<CacheControl>,
pub caching: Option<CacheControl>,
pub defer_loading: Option<bool>,
pub max_tokens: Option<u64>,
pub max_uses: Option<u64>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ToolsetToolConfig {
pub defer_loading: Option<bool>,
pub enabled: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct McpToolset {
pub mcp_server_name: String,
pub cache_control: Option<CacheControl>,
pub configs: Option<IndexMap<String, ToolsetToolConfig>>,
pub default_config: Option<ToolsetToolConfig>,
pub tools: Option<Vec<McpListedTool>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct BrowserToolsetConfigs {
#[serde(rename = "type")]
pub type_text: Option<ToolsetToolConfig>,
pub close_tab: Option<ToolsetToolConfig>,
pub double_click: Option<ToolsetToolConfig>,
pub file_upload: Option<ToolsetToolConfig>,
pub find: Option<ToolsetToolConfig>,
pub form_input: Option<ToolsetToolConfig>,
pub get_page_text: Option<ToolsetToolConfig>,
pub hold_key: Option<ToolsetToolConfig>,
pub hover: Option<ToolsetToolConfig>,
pub javascript_exec: Option<ToolsetToolConfig>,
pub key: Option<ToolsetToolConfig>,
pub left_click: Option<ToolsetToolConfig>,
pub left_click_drag: Option<ToolsetToolConfig>,
pub left_mouse_down: Option<ToolsetToolConfig>,
pub left_mouse_up: Option<ToolsetToolConfig>,
pub list_tabs: Option<ToolsetToolConfig>,
pub middle_click: Option<ToolsetToolConfig>,
pub mouse_move: Option<ToolsetToolConfig>,
pub navigate: Option<ToolsetToolConfig>,
pub new_tab: Option<ToolsetToolConfig>,
pub read_console: Option<ToolsetToolConfig>,
pub read_network: Option<ToolsetToolConfig>,
pub read_page: Option<ToolsetToolConfig>,
pub right_click: Option<ToolsetToolConfig>,
pub screenshot: Option<ToolsetToolConfig>,
pub scroll: Option<ToolsetToolConfig>,
pub scroll_to: Option<ToolsetToolConfig>,
pub switch_tab: Option<ToolsetToolConfig>,
pub triple_click: Option<ToolsetToolConfig>,
pub wait: Option<ToolsetToolConfig>,
pub zoom: Option<ToolsetToolConfig>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ComputerToolsetConfigs {
#[serde(rename = "type")]
pub type_text: Option<ToolsetToolConfig>,
pub cursor_position: Option<ToolsetToolConfig>,
pub double_click: Option<ToolsetToolConfig>,
pub hold_key: Option<ToolsetToolConfig>,
pub key: Option<ToolsetToolConfig>,
pub left_click: Option<ToolsetToolConfig>,
pub left_click_drag: Option<ToolsetToolConfig>,
pub left_mouse_down: Option<ToolsetToolConfig>,
pub left_mouse_up: Option<ToolsetToolConfig>,
pub middle_click: Option<ToolsetToolConfig>,
pub mouse_move: Option<ToolsetToolConfig>,
pub right_click: Option<ToolsetToolConfig>,
pub screenshot: Option<ToolsetToolConfig>,
pub scroll: Option<ToolsetToolConfig>,
pub triple_click: Option<ToolsetToolConfig>,
pub wait: Option<ToolsetToolConfig>,
pub zoom: Option<ToolsetToolConfig>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct Toolset<C> {
pub cache_control: Option<CacheControl>,
pub configs: Option<Box<C>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type")]
pub enum BuiltinMessagesTool {
#[serde(rename = "bash_20241022")]
Bash20241022(ClientTool<BashToolName>),
#[serde(rename = "bash_20250124")]
Bash20250124(ClientTool<BashToolName>),
#[serde(rename = "text_editor_20241022")]
TextEditor20241022(ClientTool<StrReplaceEditorName>),
#[serde(rename = "text_editor_20250124")]
TextEditor20250124(ClientTool<StrReplaceEditorName>),
#[serde(rename = "text_editor_20250429")]
TextEditor20250429(ClientTool<StrReplaceBasedEditToolName>),
#[serde(rename = "text_editor_20250728")]
TextEditor20250728(TextEditorTool20250728),
#[serde(rename = "memory_20250818")]
Memory20250818(ClientTool<MemoryToolName>),
#[serde(rename = "computer_20241022")]
Computer20241022(ComputerTool),
#[serde(rename = "computer_20250124")]
Computer20250124(ComputerTool),
#[serde(rename = "computer_20251124")]
Computer20251124(ComputerTool20251124),
#[serde(rename = "code_execution_20250522")]
CodeExecution20250522(ServerTool<CodeExecutionToolName>),
#[serde(rename = "code_execution_20250825")]
CodeExecution20250825(ServerTool<CodeExecutionToolName>),
#[serde(rename = "code_execution_20260120")]
CodeExecution20260120(ServerTool<CodeExecutionToolName>),
#[serde(rename = "code_execution_20260521")]
CodeExecution20260521(ServerTool<CodeExecutionToolName>),
#[serde(rename = "tool_search_tool_regex_20251119")]
ToolSearchRegex20251119(ServerTool<ToolSearchRegexToolName>),
#[serde(rename = "tool_search_tool_regex")]
ToolSearchRegex(ServerTool<ToolSearchRegexToolName>),
#[serde(rename = "tool_search_tool_bm25_20251119")]
ToolSearchBm2520251119(ServerTool<ToolSearchBm25ToolName>),
#[serde(rename = "tool_search_tool_bm25")]
ToolSearchBm25(ServerTool<ToolSearchBm25ToolName>),
#[serde(rename = "web_search_20250305")]
WebSearch20250305(WebSearchTool),
#[serde(rename = "web_search_20260209")]
WebSearch20260209(WebSearchTool),
#[serde(rename = "web_search_20260318")]
WebSearch20260318(WebSearchTool20260318),
#[serde(rename = "web_fetch_20250910")]
WebFetch20250910(WebFetchTool),
#[serde(rename = "web_fetch_20260209")]
WebFetch20260209(WebFetchTool),
#[serde(rename = "web_fetch_20260309")]
WebFetch20260309(WebFetchTool20260309),
#[serde(rename = "web_fetch_20260318")]
WebFetch20260318(WebFetchTool20260318),
#[serde(rename = "advisor_20260301")]
Advisor20260301(AdvisorTool),
#[serde(rename = "browser_toolset_20260801")]
BrowserToolset20260801(Toolset<BrowserToolsetConfigs>),
#[serde(rename = "computer_toolset_20260801")]
ComputerToolset20260801(Toolset<ComputerToolsetConfigs>),
#[serde(rename = "mcp_toolset")]
McpToolset(McpToolset),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum MessagesToolParam {
Builtin(Box<BuiltinMessagesTool>),
Custom(Box<CustomTool>),
}

75
litellm-rust/crates/llms-types/src/formats/messages/usage.rs Normal file
View file

@ -0,0 +1,75 @@
use serde_json::{Map, Value};
use crate::recognized::Recognized;
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ServerToolUsage {
pub web_search_requests: Option<u64>,
pub web_fetch_requests: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct UsageIteration {
#[serde(rename = "type")]
pub iteration_type: Recognized<UsageIterationType>,
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cache_creation_input_tokens: Option<u64>,
pub cache_read_input_tokens: Option<u64>,
pub cache_creation: Option<CacheCreationUsage>,
pub model: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum UsageIterationType {
Compaction,
Message,
AdvisorMessage,
FallbackMessage,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesUsage {
pub input_tokens: Option<u64>,
pub output_tokens: Option<u64>,
pub cache_creation_input_tokens: Option<u64>,
pub cache_read_input_tokens: Option<u64>,
pub server_tool_use: Option<ServerToolUsage>,
pub cache_creation: Option<CacheCreationUsage>,
pub output_tokens_details: Option<MessagesOutputTokensDetails>,
pub service_tier: Option<String>,
pub inference_geo: Option<String>,
pub speed: Option<super::Speed>,
pub iterations: Option<Vec<UsageIteration>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct CacheCreationUsage {
pub ephemeral_1h_input_tokens: Option<u64>,
pub ephemeral_5m_input_tokens: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MessagesOutputTokensDetails {
pub thinking_tokens: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

3
litellm-rust/crates/llms-types/src/formats/mod.rs
View file

@ -1,3 +1,6 @@
//! Wire types for each provider API format (OpenAI Chat Completions,
//! Anthropic Messages, Responses, ...), one submodule per format.
pub mod audio_transcription;
pub mod batches;
pub mod chat_completions;

26
litellm-rust/crates/llms-types/src/formats/ocr.rs
View file

@ -5,7 +5,7 @@ use serde_with::serde_as;
use crate::serde_compat::{FiniteF64, LaxI64};
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type")]
pub enum OcrDocument {
#[serde(rename = "document_url")]
@ -49,7 +49,7 @@ impl OcrDocument {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Default, Eq)]
#[serde(rename_all = "lowercase")]
pub enum OcrResponseFormat {
@ -59,7 +59,7 @@ pub enum OcrResponseFormat {
}
#[serde_as]
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPageDimensions {
#[serde_as(deserialize_as = "Option<LaxI64>")]
@ -70,7 +70,7 @@ pub struct OcrPageDimensions {
pub width: Option<i64>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPageImage {
pub image_base64: Option<String>,
@ -80,7 +80,7 @@ pub struct OcrPageImage {
}
#[serde_as]
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrPage {
#[serde_as(deserialize_as = "LaxI64")]
@ -93,7 +93,7 @@ pub struct OcrPage {
}
#[serde_as]
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrUsageInfo {
#[serde_as(deserialize_as = "Option<LaxI64>")]
@ -108,7 +108,7 @@ pub struct OcrUsageInfo {
pub extra_fields: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct LiteLLMOcrResponse {
pub pages: Vec<OcrPage>,
pub model: String,
@ -150,3 +150,15 @@ impl LiteLLMOcrResponse {
fn ocr_object() -> String {
"ocr".into()
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct OcrBoundingBox {
pub top_left_x: Option<serde_json::Number>,
pub top_left_y: Option<serde_json::Number>,
pub bottom_right_x: Option<serde_json::Number>,
pub bottom_right_y: Option<serde_json::Number>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

2
litellm-rust/crates/llms-types/src/formats/responses/mod.rs
View file

@ -1,4 +1,6 @@
mod output;
mod response;
pub mod streaming_websocket;
pub use output::*;
pub use response::ResponsesApiResponse;

831
litellm-rust/crates/llms-types/src/formats/responses/output.rs Normal file
View file

@ -0,0 +1,831 @@
use crate::formats::chat_completions::PromptCacheBreakpoint;
use serde_json::{Map, Value};
use std::collections::BTreeMap;
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesOutputItem {
Message(ResponsesMessage),
FunctionCall(ResponsesFunctionCall),
CustomToolCall(ResponsesCustomToolCall),
Reasoning(ResponsesReasoning),
WebSearchCall(ResponsesWebSearchCall),
FileSearchCall(ResponsesFileSearchCall),
ImageGenerationCall(ResponsesImageGenerationCall),
CodeInterpreterCall(ResponsesCodeInterpreterCall),
McpCall(ResponsesMcpCall),
McpListTools(ResponsesMcpListTools),
FunctionCallOutput(ResponsesFunctionCallOutput),
ComputerCall(ResponsesComputerCall),
ComputerCallOutput(ResponsesComputerCallOutput),
Program(ResponsesProgram),
ProgramOutput(ResponsesProgramOutput),
ToolSearchCall(ResponsesToolSearchCall),
ToolSearchOutput(ResponsesToolSearchOutput),
AdditionalTools(ResponsesAdditionalTools),
Compaction(ResponsesCompaction),
LocalShellCall(ResponsesLocalShellCall),
LocalShellCallOutput(ResponsesLocalShellCallOutput),
ShellCall(ResponsesShellCall),
ShellCallOutput(ResponsesShellCallOutput),
ApplyPatchCall(ResponsesApplyPatchCall),
ApplyPatchCallOutput(ResponsesApplyPatchCallOutput),
McpApprovalRequest(ResponsesMcpApprovalRequest),
McpApprovalResponse(ResponsesMcpApprovalResponse),
CustomToolCallOutput(ResponsesCustomToolCallOutput),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesContentPart {
OutputText {
text: String,
#[serde(skip_serializing_if = "Option::is_none")]
annotations: Option<Vec<ResponsesAnnotation>>,
#[serde(skip_serializing_if = "Option::is_none")]
logprobs: Option<Vec<crate::formats::chat_completions::ChatTokenLogprob>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Refusal {
refusal: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
SummaryText {
text: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
ReasoningText {
text: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesAnnotation {
UrlCitation(ResponsesUrlCitation),
FileCitation(ResponsesFileCitation),
FilePath(ResponsesFileCitation),
ContainerFileCitation(ResponsesFileCitation),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesCodeOutput {
Logs {
logs: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
Image {
url: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMessage {
pub id: Option<String>,
pub status: Option<String>,
pub role: Option<String>,
pub phase: Option<String>,
pub content: Option<Vec<ResponsesContentPart>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesFunctionCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub name: Option<String>,
pub arguments: Option<String>,
pub namespace: Option<String>,
pub r#async: Option<bool>,
pub caller: Option<ResponsesToolCaller>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesCustomToolCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub name: Option<String>,
pub input: Option<String>,
pub namespace: Option<String>,
pub r#async: Option<bool>,
pub caller: Option<ResponsesToolCaller>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesReasoning {
pub id: Option<String>,
pub status: Option<String>,
pub summary: Option<Vec<ResponsesContentPart>>,
pub content: Option<Vec<ResponsesContentPart>>,
pub encrypted_content: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesWebSearchCall {
pub id: Option<String>,
pub status: Option<String>,
pub action: Option<ResponsesWebSearchAction>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesFileSearchCall {
pub id: Option<String>,
pub status: Option<String>,
pub queries: Option<Vec<String>>,
pub results: Option<Vec<ResponsesFileSearchResult>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesFileSearchResult {
pub file_id: Option<String>,
pub filename: Option<String>,
pub score: Option<serde_json::Number>,
pub text: Option<String>,
pub attributes: Option<Map<String, Value>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesImageGenerationCall {
pub id: Option<String>,
pub status: Option<String>,
pub result: Option<String>,
pub action: Option<String>,
pub background: Option<String>,
pub output_format: Option<String>,
pub quality: Option<String>,
pub revised_prompt: Option<String>,
pub size: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesCodeInterpreterCall {
pub id: Option<String>,
pub status: Option<String>,
pub code: Option<String>,
pub container_id: Option<String>,
pub outputs: Option<Vec<ResponsesCodeOutput>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMcpCall {
pub id: Option<String>,
pub status: Option<String>,
pub name: Option<String>,
pub server_label: Option<String>,
pub arguments: Option<String>,
pub output: Option<String>,
pub error: Option<ResponsesMcpError>,
pub approval_request_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesUrlCitation {
pub url: Option<String>,
pub title: Option<String>,
pub start_index: Option<u64>,
pub end_index: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesFileCitation {
pub file_id: Option<String>,
pub filename: Option<String>,
pub index: Option<u64>,
pub start_index: Option<u64>,
pub end_index: Option<u64>,
pub container_id: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesWebSearchAction {
Search {
#[serde(skip_serializing_if = "Option::is_none")]
query: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
queries: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
sources: Option<Vec<ResponsesWebSearchSource>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
OpenPage {
#[serde(skip_serializing_if = "Option::is_none")]
url: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
#[serde(alias = "find")]
FindInPage {
url: String,
pattern: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesWebSearchSource {
#[serde(rename = "type")]
pub source_type: Option<String>,
pub url: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMcpListTools {
pub id: Option<String>,
pub server_label: Option<String>,
pub tools: Option<Vec<ResponsesMcpTool>>,
pub error: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMcpTool {
pub name: Option<String>,
pub description: Option<String>,
pub input_schema: Option<crate::json_schema::JsonSchema>,
pub annotations: Option<Value>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ResponsesMcpError {
Message(String),
Detail(ResponsesMcpErrorDetail),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesMcpErrorDetail {
McpProtocolError {
code: i64,
message: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
HttpError {
code: i64,
message: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
McpToolExecutionError {
content: Value,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesToolCaller {
Direct {
#[serde(flatten)]
extra: Map<String, Value>,
},
Program {
caller_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ResponsesToolOutput {
Text(String),
Content(Vec<ResponsesInputContent>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesInputContent {
InputText {
text: String,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
InputImage {
detail: String,
#[serde(skip_serializing_if = "Option::is_none")]
file_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
image_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
InputFile {
#[serde(skip_serializing_if = "Option::is_none")]
detail: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
file_data: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
file_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
file_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
filename: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
prompt_cache_breakpoint: Option<PromptCacheBreakpoint>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesFunctionCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub output: Option<ResponsesToolOutput>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
pub name: Option<String>,
pub namespace: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesCustomToolCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub output: Option<ResponsesToolOutput>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesSafetyCheck {
pub id: Option<String>,
pub code: Option<String>,
pub message: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesComputerCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub pending_safety_checks: Option<Vec<ResponsesSafetyCheck>>,
pub action: Option<ResponsesComputerAction>,
pub actions: Option<Vec<ResponsesComputerAction>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesComputerAction {
Click {
button: String,
x: i64,
y: i64,
#[serde(skip_serializing_if = "Option::is_none")]
keys: Option<Vec<String>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
DoubleClick {
x: i64,
y: i64,
#[serde(skip_serializing_if = "Option::is_none")]
keys: Option<Vec<String>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Drag {
path: Vec<ResponsesCoordinate>,
#[serde(skip_serializing_if = "Option::is_none")]
keys: Option<Vec<String>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Keypress {
keys: Vec<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Move {
x: i64,
y: i64,
#[serde(skip_serializing_if = "Option::is_none")]
keys: Option<Vec<String>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Screenshot {
#[serde(flatten)]
extra: Map<String, Value>,
},
Scroll {
scroll_x: i64,
scroll_y: i64,
x: i64,
y: i64,
#[serde(skip_serializing_if = "Option::is_none")]
keys: Option<Vec<String>>,
#[serde(flatten)]
extra: Map<String, Value>,
},
Type {
text: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
Wait {
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ResponsesCoordinate {
pub x: i64,
pub y: i64,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesComputerCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub output: Option<ResponsesComputerOutput>,
pub acknowledged_safety_checks: Option<Vec<ResponsesSafetyCheck>>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesComputerOutput {
ComputerScreenshot {
#[serde(skip_serializing_if = "Option::is_none")]
file_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
image_url: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesProgram {
pub id: Option<String>,
pub call_id: Option<String>,
pub code: Option<String>,
pub fingerprint: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesProgramOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub result: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesToolSearchCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub arguments: Option<Value>,
pub execution: Option<String>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesToolSearchOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub execution: Option<String>,
pub tools: Option<Vec<Map<String, Value>>>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesAdditionalTools {
pub id: Option<String>,
pub role: Option<String>,
pub tools: Option<Vec<Map<String, Value>>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesCompaction {
pub id: Option<String>,
pub encrypted_content: Option<String>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesLocalShellCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub action: Option<ResponsesLocalShellAction>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesLocalShellAction {
Exec {
command: Vec<String>,
env: BTreeMap<String, String>,
#[serde(skip_serializing_if = "Option::is_none")]
timeout_ms: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
user: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
working_directory: Option<String>,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesLocalShellCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub output: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesShellCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub action: Option<ResponsesShellAction>,
pub environment: Option<ResponsesShellEnvironment>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesShellAction {
pub commands: Option<Vec<String>>,
pub max_output_length: Option<u64>,
pub timeout_ms: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesShellEnvironment {
Local {
#[serde(flatten)]
extra: Map<String, Value>,
},
ContainerReference {
container_id: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesShellCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub max_output_length: Option<u64>,
pub output: Option<Vec<ResponsesShellOutputChunk>>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesShellOutputChunk {
pub outcome: Option<ResponsesShellOutcome>,
pub stdout: Option<String>,
pub stderr: Option<String>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesShellOutcome {
Timeout {
#[serde(flatten)]
extra: Map<String, Value>,
},
Exit {
exit_code: i64,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesApplyPatchCall {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub operation: Option<ResponsesApplyPatchOperation>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ResponsesApplyPatchOperation {
CreateFile {
path: String,
diff: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
DeleteFile {
path: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
UpdateFile {
path: String,
diff: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesApplyPatchCallOutput {
pub id: Option<String>,
pub status: Option<String>,
pub call_id: Option<String>,
pub output: Option<String>,
pub caller: Option<ResponsesToolCaller>,
pub created_by: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMcpApprovalRequest {
pub id: Option<String>,
pub server_label: Option<String>,
pub name: Option<String>,
pub arguments: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesMcpApprovalResponse {
pub id: Option<String>,
pub approval_request_id: Option<String>,
pub approve: Option<bool>,
pub reason: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

2
litellm-rust/crates/llms-types/src/formats/responses/response.rs
View file

@ -1,6 +1,6 @@
use serde_json::{Map, Value};
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ResponsesApiResponse {
pub id: String,
pub model: String,

20
litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs
View file

@ -1,3 +1,5 @@
use super::ResponsesOutputItem;
use crate::recognized::Recognized;
use serde_json::{Map, Value};
#[derive(
@ -36,7 +38,7 @@ impl ResponsesWsEventType {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct ResponsesWsEvent {
#[serde(rename = "type")]
pub event_type: ResponsesWsEventType,
@ -62,7 +64,7 @@ impl ResponsesWsEvent {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq)]
pub struct ResponsesErrorFrame {
#[serde(rename = "type")]
@ -82,7 +84,7 @@ impl ResponsesErrorFrame {
}
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq)]
pub struct ResponsesErrorBody {
#[serde(rename = "type")]
@ -90,6 +92,18 @@ pub struct ResponsesErrorBody {
pub message: String,
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ResponsesEventResponse {
pub id: Option<String>,
pub model: Option<String>,
pub status: Option<String>,
pub output: Option<Vec<Recognized<ResponsesOutputItem>>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[cfg(test)]
mod tests {
use rstest::rstest;

4
litellm-rust/crates/llms-types/src/headers.rs
View file

@ -1,6 +1,6 @@
use serde_json::{Map, Value};
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct ProviderSpecificHeader {
#[serde(default)]
@ -9,7 +9,7 @@ pub struct ProviderSpecificHeader {
pub extra_headers: Map<String, Value>,
}
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum ProviderSpecificHeaders {
One(ProviderSpecificHeader),

51
litellm-rust/crates/llms-types/src/json_schema.rs Normal file
View file

@ -0,0 +1,51 @@
use indexmap::IndexMap;
use serde_json::{Map, Value};
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum JsonSchema {
Boolean(bool),
Object(Box<JsonSchemaObject>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum JsonSchemaType {
Name(String),
Names(Vec<String>),
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum JsonSchemaItems {
Schema(Box<JsonSchema>),
Tuple(Vec<JsonSchema>),
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct JsonSchemaObject {
#[serde(rename = "type")]
pub schema_type: Option<JsonSchemaType>,
pub properties: Option<IndexMap<String, JsonSchema>>,
pub required: Option<Vec<String>>,
#[serde(rename = "additionalProperties")]
pub additional_properties: Option<Box<JsonSchema>>,
pub items: Option<JsonSchemaItems>,
#[serde(rename = "prefixItems")]
pub prefix_items: Option<Vec<JsonSchema>>,
#[serde(rename = "$defs")]
pub defs: Option<IndexMap<String, JsonSchema>>,
#[serde(rename = "$ref")]
pub reference: Option<String>,
#[serde(rename = "anyOf")]
pub any_of: Option<Vec<JsonSchema>>,
#[serde(rename = "allOf")]
pub all_of: Option<Vec<JsonSchema>>,
#[serde(rename = "oneOf")]
pub one_of: Option<Vec<JsonSchema>>,
pub strict: Option<bool>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

1
litellm-rust/crates/llms-types/src/lib.rs
View file

@ -6,6 +6,7 @@ macro_rules_attribute::attribute_alias! {
pub mod formats;
pub mod headers;
pub mod json_schema;
pub mod providers;
pub mod recognized;
pub mod serde_compat;

18
litellm-rust/crates/llms-types/src/providers/AGENTS.md Normal file
View file

@ -0,0 +1,18 @@
# references
These links describe the provider-specific wire types in this directory. Shared Messages payload references live in `../formats/messages/AGENTS.md`
## anthropic.rs
`AnthropicBeta`, `BetaSet` and `BetaProvider` represent the `anthropic-beta` header values, their
wire spelling and per-host support
- https://platform.claude.com/docs/en/api/beta-headers.md
## minimax.rs
MiniMax's Messages reference documents its image, video, and mid-conversation system content extensions. Its cache reference documents `cache_control` on those blocks
- https://platform.minimax.io/docs/api-reference/text-chat-anthropic
- https://platform.minimax.io/docs/api-reference/text-chat-anthropic.md
- https://platform.minimax.io/docs/api-reference/anthropic-api-compatible-cache.md

386
litellm-rust/crates/llms-types/src/providers/anthropic.rs
View file

@ -7,6 +7,39 @@ use std::{
str::FromStr,
};
use strum::VariantArray;
/// A provider column of `litellm/anthropic_beta_headers_config.json`: which betas a host accepts
/// and under which name.
#[derive(
Clone,
Copy,
Debug,
PartialEq,
Eq,
Hash,
strum::AsRefStr,
strum::Display,
strum::EnumString,
strum::VariantArray,
)]
pub enum BetaProvider {
#[strum(serialize = "anthropic")]
Anthropic,
#[strum(serialize = "azure_ai")]
AzureAi,
#[strum(serialize = "bedrock_converse")]
BedrockConverse,
#[strum(serialize = "bedrock")]
Bedrock,
#[strum(serialize = "bedrock_mantle")]
BedrockMantle,
#[strum(serialize = "vertex_ai")]
VertexAi,
#[strum(serialize = "databricks")]
Databricks,
}
/// One value of the `anthropic-beta` header. Equality, ordering and hashing follow the wire
/// string, so a value parsed from a caller's header never disagrees with the matching variant.
#[derive(Clone, Debug, strum::AsRefStr, strum::Display, strum::EnumString)]
@ -35,12 +68,68 @@ pub enum AnthropicBeta {
PerTurnControl20260701,
#[strum(serialize = "dangerous-tool-use-2026-09-03")]
DangerousToolUse20260903,
#[strum(serialize = "bash_20241022")]
Bash20241022,
#[strum(serialize = "bash_20250124")]
Bash20250124,
#[strum(serialize = "claude-code-20250219")]
ClaudeCode20250219,
#[strum(serialize = "code-execution-2025-08-25")]
CodeExecution20250825,
#[strum(serialize = "computer-use-2025-01-24")]
ComputerUse20250124,
#[strum(serialize = "computer-use-2025-11-24")]
ComputerUse20251124,
#[strum(serialize = "context-1m-2025-08-07")]
Context1m20250807,
#[strum(serialize = "effort-2025-11-24")]
Effort20251124,
#[strum(serialize = "files-api-2025-04-14")]
FilesApi20250414,
#[strum(serialize = "fine-grained-tool-streaming-2025-05-14")]
FineGrainedToolStreaming20250514,
#[strum(serialize = "interleaved-thinking-2025-05-14")]
InterleavedThinking20250514,
#[strum(serialize = "mcp-client-2025-04-04")]
McpClient20250404,
#[strum(serialize = "mcp-client-2025-11-20")]
McpClient20251120,
#[strum(serialize = "mcp-servers-2025-12-04")]
McpServers20251204,
#[strum(serialize = "mid-conversation-output-config-2026-07-01")]
MidConversationOutputConfig20260701,
#[strum(serialize = "mid-conversation-tool-changes-2026-07-01")]
MidConversationToolChanges20260701,
#[strum(serialize = "output-128k-2025-02-19")]
Output128k20250219,
#[strum(serialize = "prompt-caching-scope-2026-01-05")]
PromptCachingScope20260105,
#[strum(serialize = "skills-2025-10-02")]
Skills20251002,
#[strum(serialize = "structured-output-2024-03-01")]
StructuredOutput20240301,
#[strum(serialize = "text_editor_20241022")]
TextEditor20241022,
#[strum(serialize = "text_editor_20250124")]
TextEditor20250124,
#[strum(serialize = "thinking-binding-controls-2026-08-01")]
ThinkingBindingControls20260801,
#[strum(serialize = "thinking-display-updates-2026-08-18")]
ThinkingDisplayUpdates20260818,
#[strum(serialize = "token-efficient-tools-2025-02-19")]
TokenEfficientTools20250219,
#[strum(serialize = "tool-examples-2025-10-29")]
ToolExamples20251029,
#[strum(serialize = "tool-search-tool-2025-10-19")]
ToolSearchTool20251019,
#[strum(serialize = "inline-tools-2026-09-15")]
InlineTools20260915,
#[strum(default, transparent)]
Other(String),
}
impl AnthropicBeta {
pub const KNOWN: [Self; 12] = [
pub const KNOWN: [Self; 40] = [
Self::Oauth20250420,
Self::WebFetch20250910,
Self::WebSearch20250305,
@ -53,11 +142,153 @@ impl AnthropicBeta {
Self::AdvisorTool20260301,
Self::PerTurnControl20260701,
Self::DangerousToolUse20260903,
Self::Bash20241022,
Self::Bash20250124,
Self::ClaudeCode20250219,
Self::CodeExecution20250825,
Self::ComputerUse20250124,
Self::ComputerUse20251124,
Self::Context1m20250807,
Self::Effort20251124,
Self::FilesApi20250414,
Self::FineGrainedToolStreaming20250514,
Self::InterleavedThinking20250514,
Self::McpClient20250404,
Self::McpClient20251120,
Self::McpServers20251204,
Self::MidConversationOutputConfig20260701,
Self::MidConversationToolChanges20260701,
Self::Output128k20250219,
Self::PromptCachingScope20260105,
Self::Skills20251002,
Self::StructuredOutput20240301,
Self::TextEditor20241022,
Self::TextEditor20250124,
Self::ThinkingBindingControls20260801,
Self::ThinkingDisplayUpdates20260818,
Self::TokenEfficientTools20250219,
Self::ToolExamples20251029,
Self::ToolSearchTool20251019,
Self::InlineTools20260915,
];
pub fn as_str(&self) -> &str {
self.as_ref()
}
pub fn on(&self, provider: BetaProvider) -> Option<Self> {
let resolved = match self {
Self::Other(raw) => raw.parse().unwrap(),
_ => self.clone(),
};
if resolved.rejected_by().contains(&provider) {
return None;
}
Some(match (resolved, provider) {
(
Self::AdvancedToolUse20251120,
BetaProvider::Bedrock | BetaProvider::BedrockMantle | BetaProvider::VertexAi,
) => Self::ToolSearchTool20251019,
(beta, _) => beta,
})
}
fn rejected_by(&self) -> &'static [BetaProvider] {
match self {
Self::AdvancedToolUse20251120 | Self::ContextManagement20250627 => {
&[BetaProvider::BedrockConverse]
}
Self::AdvisorTool20260301 | Self::Compact20260904 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Bash20241022
| Self::Bash20250124
| Self::McpServers20251204
| Self::StructuredOutput20240301
| Self::TextEditor20241022
| Self::TextEditor20250124 => BetaProvider::VARIANTS,
Self::ClaudeCode20250219 | Self::ToolExamples20251029 => &[
BetaProvider::Anthropic,
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::CodeExecution20250825
| Self::FilesApi20250414
| Self::McpClient20250404
| Self::McpClient20251120
| Self::PromptCachingScope20260105
| Self::Skills20251002
| Self::WebFetch20250910 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
Self::DangerousToolUse20260903 => {
&[BetaProvider::BedrockConverse, BetaProvider::Databricks]
}
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
],
Self::FineGrainedToolStreaming20250514 => {
&[BetaProvider::AzureAi, BetaProvider::VertexAi]
}
Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
&[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
}
Self::MidConversationOutputConfig20260701
| Self::MidConversationToolChanges20260701 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::BedrockMantle,
BetaProvider::VertexAi,
BetaProvider::Databricks,
],
Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::VertexAi,
],
Self::PerTurnControl20260701 => &[
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Databricks,
],
Self::InlineTools20260915 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
BetaProvider::BedrockMantle,
BetaProvider::Databricks,
],
Self::Other(_) => BetaProvider::VARIANTS,
}
}
}
impl PartialEq for AnthropicBeta {
@ -149,10 +380,20 @@ impl fmt::Display for BetaSet {
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use indexmap::IndexMap;
use rstest::rstest;
use super::*;
fn beta_headers_config() -> IndexMap<String, serde_json::Value> {
serde_json::from_str(include_str!(
"../../../../../litellm/anthropic_beta_headers_config.json"
))
.unwrap()
}
fn set(header: &str) -> BetaSet {
header.parse().unwrap_or_else(|never| match never {})
}
@ -171,7 +412,35 @@ mod tests {
AnthropicBeta::FastMode20260201,
AnthropicBeta::AdvisorTool20260301,
AnthropicBeta::PerTurnControl20260701,
AnthropicBeta::DangerousToolUse20260903
AnthropicBeta::DangerousToolUse20260903,
AnthropicBeta::Bash20241022,
AnthropicBeta::Bash20250124,
AnthropicBeta::ClaudeCode20250219,
AnthropicBeta::CodeExecution20250825,
AnthropicBeta::ComputerUse20250124,
AnthropicBeta::ComputerUse20251124,
AnthropicBeta::Context1m20250807,
AnthropicBeta::Effort20251124,
AnthropicBeta::FilesApi20250414,
AnthropicBeta::FineGrainedToolStreaming20250514,
AnthropicBeta::InterleavedThinking20250514,
AnthropicBeta::McpClient20250404,
AnthropicBeta::McpClient20251120,
AnthropicBeta::McpServers20251204,
AnthropicBeta::MidConversationOutputConfig20260701,
AnthropicBeta::MidConversationToolChanges20260701,
AnthropicBeta::Output128k20250219,
AnthropicBeta::PromptCachingScope20260105,
AnthropicBeta::Skills20251002,
AnthropicBeta::StructuredOutput20240301,
AnthropicBeta::TextEditor20241022,
AnthropicBeta::TextEditor20250124,
AnthropicBeta::ThinkingBindingControls20260801,
AnthropicBeta::ThinkingDisplayUpdates20260818,
AnthropicBeta::TokenEfficientTools20250219,
AnthropicBeta::ToolExamples20251029,
AnthropicBeta::ToolSearchTool20251019,
AnthropicBeta::InlineTools20260915
)]
beta: AnthropicBeta,
) {
@ -181,6 +450,119 @@ mod tests {
assert!(AnthropicBeta::KNOWN.contains(&beta));
}
#[rstest]
fn known_betas_are_exactly_the_config_keys() {
let config = beta_headers_config();
let config_keys: BTreeSet<String> = config
.iter()
.filter(|(provider, _)| provider.as_str() != "description")
.flat_map(|(_, column)| {
column
.as_object()
.into_iter()
.flat_map(|values| values.keys().cloned())
})
.collect();
let known_keys: BTreeSet<String> = AnthropicBeta::KNOWN
.iter()
.map(|beta| beta.as_str().to_string())
.collect();
assert_eq!(known_keys, config_keys);
}
#[rstest]
#[case::anthropic(BetaProvider::Anthropic)]
#[case::azure_ai(BetaProvider::AzureAi)]
#[case::bedrock_converse(BetaProvider::BedrockConverse)]
#[case::bedrock(BetaProvider::Bedrock)]
#[case::bedrock_mantle(BetaProvider::BedrockMantle)]
#[case::vertex_ai(BetaProvider::VertexAi)]
#[case::databricks(BetaProvider::Databricks)]
fn provider_columns_match_beta_provider_variants(#[case] provider: BetaProvider) {
let config = beta_headers_config();
let config_columns: Vec<String> = config
.keys()
.filter(|column| column.as_str() != "description")
.cloned()
.collect();
let beta_provider_columns: Vec<String> = BetaProvider::VARIANTS
.iter()
.map(ToString::to_string)
.collect();
assert_eq!(config_columns, beta_provider_columns);
assert_eq!(
provider.to_string().parse::<BetaProvider>().unwrap(),
provider
);
}
#[rstest]
fn on_matches_every_config_cell() {
let config = beta_headers_config();
for provider in BetaProvider::VARIANTS {
for beta in &AnthropicBeta::KNOWN {
let expected = config
.get(provider.as_ref())
.and_then(serde_json::Value::as_object)
.and_then(|column| column.get(beta.as_str()))
.and_then(serde_json::Value::as_str)
.map(str::to_string);
let actual = beta.on(*provider).map(|name| name.to_string());
assert_eq!(actual, expected, "provider {provider}, beta {beta}");
if matches!(beta, AnthropicBeta::AdvancedToolUse20251120)
&& matches!(
provider,
BetaProvider::Bedrock
| BetaProvider::BedrockMantle
| BetaProvider::VertexAi
)
{
let parsed: AnthropicBeta = actual.as_deref().unwrap().parse().unwrap();
assert!(AnthropicBeta::KNOWN.contains(&parsed));
assert!(!matches!(parsed, AnthropicBeta::Other(_)));
}
}
}
}
#[rstest]
#[case::renamed(
"advanced-tool-use-2025-11-20",
BetaProvider::Bedrock,
Some(AnthropicBeta::ToolSearchTool20251019)
)]
#[case::kept(
"oauth-2025-04-20",
BetaProvider::Anthropic,
Some(AnthropicBeta::Oauth20250420)
)]
#[case::rejected("effort-2025-11-24", BetaProvider::VertexAi, None)]
#[case::unknown("example-beta-2099-01-01", BetaProvider::Anthropic, None)]
fn on_resolves_known_spellings_held_as_other(
#[case] raw: &str,
#[case] provider: BetaProvider,
#[case] expected: Option<AnthropicBeta>,
) {
let actual = AnthropicBeta::Other(raw.to_string()).on(provider);
assert_eq!(actual, expected);
assert!(!matches!(actual, Some(AnthropicBeta::Other(_))));
}
#[rstest]
#[case::anthropic(BetaProvider::Anthropic)]
#[case::azure_ai(BetaProvider::AzureAi)]
#[case::bedrock_converse(BetaProvider::BedrockConverse)]
#[case::bedrock(BetaProvider::Bedrock)]
#[case::bedrock_mantle(BetaProvider::BedrockMantle)]
#[case::vertex_ai(BetaProvider::VertexAi)]
#[case::databricks(BetaProvider::Databricks)]
fn unknown_beta_is_rejected_by_every_provider(#[case] provider: BetaProvider) {
assert_eq!(
AnthropicBeta::Other("example-beta-2099-01-01".into()).on(provider),
None
);
}
#[test]
fn unknown_values_are_kept_verbatim() {
let parsed: AnthropicBeta = "claude-code-20250219".parse().unwrap();

59
litellm-rust/crates/llms-types/src/providers/minimax.rs Normal file
View file

@ -0,0 +1,59 @@
use serde_json::{Map, Value};
use crate::formats::messages::CacheControl;
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MinimaxMessagesContentBlock {
Image(MinimaxMediaBlock),
Video(MinimaxMediaBlock),
MidConvSystem {
text: String,
#[serde(flatten)]
extra: Map<String, Value>,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct MinimaxMediaBlock {
pub source: MinimaxMediaSource,
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MinimaxMediaSource {
Base64 {
media_type: String,
data: String,
#[serde(flatten)]
options: MinimaxMediaOptions,
},
Url {
url: String,
#[serde(flatten)]
options: MinimaxMediaOptions,
},
}
#[serde_with::skip_serializing_none]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Default)]
pub struct MinimaxMediaOptions {
pub detail: Option<MinimaxMediaDetail>,
pub fps: Option<serde_json::Number>,
pub max_long_side_pixel: Option<u64>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(rename_all = "snake_case")]
pub enum MinimaxMediaDetail {
Low,
Default,
High,
}

1
litellm-rust/crates/llms-types/src/providers/mod.rs
View file

@ -1 +1,2 @@
pub mod anthropic;
pub mod minimax;

2
litellm-rust/crates/llms-types/src/recognized.rs
View file

@ -1,6 +1,6 @@
use serde_json::Value;
#[macro_rules_attribute::apply(wire_type)]
#[macro_rules_attribute::apply(crate::wire_type)]
#[serde(untagged)]
pub enum Recognized<T> {
Known(T),

163
litellm-rust/crates/llms-types/tests/chat.rs Normal file
View file

@ -0,0 +1,163 @@
use litellm_llms_types::formats::chat_completions::{
ChatContentPart, ChatLogprobs, ChatMediaUrl, PromptCacheMode,
};
use litellm_llms_types::formats::messages::ContentSource;
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::parts(json!([
{"type":"text","text":"hello","cache_control":{"type":"ephemeral"},"prompt_cache_breakpoint":{"mode":"explicit"}},
{"type":"image_url","image_url":{"url":"https://example.test/image","detail":"high","format":"image/png"}},
{"type":"video_url","video_url":"https://example.test/video"},
{"type":"input_audio","input_audio":{"data":"AA==","format":"wav"}},
{"type":"file","file":{"file_id":"file_1","file_data":"JVBE","filename":"a.mp4","format":"video/mp4","detail":"low","video_metadata":{"fps":1,"start_offset":"1s","end_offset":"2s"}}},
{"type":"document","source":{"type":"text","media_type":"text/plain","data":"doc"},"title":"T","context":"C","citations":{"enabled":true}},
{"type":"refusal","refusal":"refused"}
]))]
fn content_parts_round_trip(#[case] wire: Value) {
let parts: Vec<ChatContentPart> = serde_json::from_value(wire.clone()).unwrap();
let [
ChatContentPart::Text {
text,
cache_control,
prompt_cache_breakpoint: Some(breakpoint),
extra: text_extra,
},
ChatContentPart::ImageUrl {
image_url: ChatMediaUrl::Parameters(image),
prompt_cache_breakpoint: None,
..
},
ChatContentPart::VideoUrl {
video_url: ChatMediaUrl::Url(video),
..
},
ChatContentPart::InputAudio { input_audio, .. },
ChatContentPart::File { file, .. },
ChatContentPart::Document {
source,
title: Some(title),
context: Some(context),
citations: Some(citations),
..
},
ChatContentPart::Refusal { refusal, .. },
] = parts.as_slice()
else {
panic!("expected typed content parts");
};
assert_eq!(text, "hello");
assert_eq!(
cache_control.as_ref().unwrap().cache_type.as_deref(),
Some("ephemeral")
);
assert_eq!(breakpoint.mode, PromptCacheMode::Explicit);
assert!(breakpoint.extra.is_empty());
assert!(text_extra.is_empty());
assert_eq!(image.url, "https://example.test/image");
assert_eq!(image.detail.as_deref(), Some("high"));
assert_eq!(image.format.as_deref(), Some("image/png"));
assert!(image.extra.is_empty());
assert_eq!(video, "https://example.test/video");
assert_eq!(input_audio.data, "AA==");
assert_eq!(input_audio.format, "wav");
assert_eq!(file.file_id.as_deref(), Some("file_1"));
assert_eq!(file.file_data.as_deref(), Some("JVBE"));
assert_eq!(file.filename.as_deref(), Some("a.mp4"));
assert_eq!(file.format.as_deref(), Some("video/mp4"));
assert_eq!(file.detail.as_deref(), Some("low"));
assert!(file.extra.is_empty());
let metadata = file.video_metadata.as_ref().unwrap();
assert_eq!(metadata.fps, Some(1.into()));
assert_eq!(metadata.start_offset.as_deref(), Some("1s"));
assert_eq!(metadata.end_offset.as_deref(), Some("2s"));
assert!(metadata.extra.is_empty());
let ContentSource::Text {
data, media_type, ..
} = source.as_ref()
else {
panic!("expected document text source");
};
assert_eq!(data, "doc");
assert_eq!(media_type, "text/plain");
assert_eq!(title, "T");
assert_eq!(context, "C");
assert_eq!(citations.enabled, Some(true));
assert_eq!(refusal, "refused");
assert_eq!(serde_json::to_value(parts).unwrap(), wire);
}
#[rstest]
fn logprobs_round_trip_with_tokens_and_alternatives() {
let wire = json!({
"content":[{"token":"hi","logprob":-1,"bytes":[104,105],"top_logprobs":[{"token":"hey","logprob":-2.5,"bytes":[104]}]}],
"refusal":[{"token":"refused","logprob":-3}]
});
let logprobs: ChatLogprobs = serde_json::from_value(wire.clone()).unwrap();
let token = &logprobs.content.as_ref().unwrap()[0];
assert_eq!(token.token, "hi");
assert_eq!(token.logprob, (-1).into());
assert_eq!(token.bytes.as_deref(), Some([104, 105].as_slice()));
let alternative = &token.top_logprobs.as_ref().unwrap()[0];
assert_eq!(alternative.token, "hey");
assert_eq!(alternative.bytes.as_deref(), Some([104].as_slice()));
assert_eq!(logprobs.refusal.as_ref().unwrap()[0].token, "refused");
assert!(logprobs.refusal.as_ref().unwrap()[0].top_logprobs.is_none());
assert_eq!(serde_json::to_value(logprobs).unwrap(), wire);
}
#[rstest]
#[case::text(json!({"type":"text","text":false}))]
#[case::missing_image(json!({"type":"image_url"}))]
#[case::audio_shape(json!({"type":"input_audio","input_audio":{"data":7,"format":"wav"}}))]
#[case::audio_missing_format(json!({"type":"input_audio","input_audio":{"data":"AA=="}}))]
#[case::image_missing_url(json!({"type":"image_url","image_url":{"detail":"high"}}))]
#[case::file_metadata(json!({"type":"file","file":{"video_metadata":{"fps":"fast"}}}))]
#[case::document_source(json!({"type":"document","source":{"type":"url","url":false}}))]
#[case::refusal_missing_text(json!({"type":"refusal"}))]
#[case::file_missing_file(json!({"type":"file"}))]
#[case::document_missing_source(json!({"type":"document"}))]
#[case::video_missing_url(json!({"type":"video_url"}))]
#[case::cache_control_shape(json!({"type":"text","text":"t","cache_control":"ephemeral"}))]
#[case::breakpoint_missing_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{}}))]
#[case::breakpoint_unknown_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{"mode":"auto"}}))]
#[case::unknown_tag(json!({"type":"future"}))]
fn content_parts_reject_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ChatContentPart>(wire).is_err());
}
#[rstest]
fn partial_file_preserves_extensions_and_omits_null_optionals() {
let wire = json!({"type":"file","file":{"file_id":null,"filename":"a.pdf","extension":[1,null]},"future":true});
let part: ChatContentPart = serde_json::from_value(wire).unwrap();
let ChatContentPart::File {
file,
prompt_cache_breakpoint: None,
extra,
} = &part
else {
panic!("expected file")
};
assert!(file.file_id.is_none());
assert!(file.file_data.is_none());
assert_eq!(file.filename.as_deref(), Some("a.pdf"));
assert_eq!(extra["future"], json!(true));
assert_eq!(
serde_json::to_value(part).unwrap(),
json!({"type":"file","file":{"filename":"a.pdf","extension":[1,null]},"future":true})
);
}
#[rstest]
#[case::missing_token(json!({"logprob":-1}))]
#[case::missing_logprob(json!({"token":"hi"}))]
#[case::wrong_bytes(json!({"token":"hi","logprob":-1,"bytes":[256]}))]
fn token_logprobs_reject_malformed_fields(#[case] wire: Value) {
assert!(
serde_json::from_value::<litellm_llms_types::formats::chat_completions::ChatTokenLogprob>(
wire
)
.is_err()
);
}

148
litellm-rust/crates/llms-types/tests/json_schema.rs Normal file
View file

@ -0,0 +1,148 @@
use litellm_llms_types::json_schema::{
JsonSchema, JsonSchemaItems, JsonSchemaObject, JsonSchemaType,
};
use rstest::rstest;
use serde::{Serialize, de::DeserializeOwned};
use serde_json::{Value, json};
fn round_trip<T>(wire: Value) -> T
where
T: DeserializeOwned + Serialize,
{
let parsed: T = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(&parsed).unwrap(), wire);
parsed
}
#[rstest]
#[case::boolean(json!(false))]
#[case::nested(json!({
"type":["object","null"],
"properties":{"nested":{"type":"array","items":{"$ref":"#/$defs/item"}},"free":true},
"additionalProperties":{"type":"string"},
"$defs":{"item":{"anyOf":[{"type":"string"},false]}},
"required":["nested"],
"enum":[{"custom":[1,null]}],
"future":null
}))]
fn recursive_schema_round_trips(#[case] wire: Value) {
let schema = round_trip::<JsonSchema>(wire);
match schema {
JsonSchema::Boolean(allowed) => assert!(!allowed),
JsonSchema::Object(schema) => {
assert_eq!(
schema.schema_type,
Some(JsonSchemaType::Names(vec!["object".into(), "null".into()]))
);
let properties = schema.properties.as_ref().unwrap();
let JsonSchema::Object(nested) = &properties["nested"] else {
panic!("expected nested schema")
};
let Some(JsonSchemaItems::Schema(items)) = &nested.items else {
panic!("expected single items schema")
};
let JsonSchema::Object(items) = items.as_ref() else {
panic!("expected items schema")
};
assert_eq!(items.reference.as_deref(), Some("#/$defs/item"));
assert_eq!(properties["free"], JsonSchema::Boolean(true));
let JsonSchema::Object(definition) = &schema.defs.as_ref().unwrap()["item"] else {
panic!("expected schema definition")
};
assert_eq!(
definition.any_of.as_ref().unwrap()[1],
JsonSchema::Boolean(false)
);
assert_eq!(schema.extra["enum"], json!([{"custom":[1,null]}]));
}
}
}
#[rstest]
fn schema_maps_keep_wire_key_order() {
let wire = r#"{"type":"object","properties":{"zeta":{"type":"string"},"alpha":{"type":"integer"}},"$defs":{"y":true,"b":false}}"#;
let schema: JsonSchemaObject = serde_json::from_str(wire).unwrap();
let names: Vec<&str> = schema
.properties
.as_ref()
.unwrap()
.keys()
.map(String::as_str)
.collect();
assert_eq!(names, ["zeta", "alpha"]);
assert_eq!(serde_json::to_string(&schema).unwrap(), wire);
}
#[rstest]
fn schema_object_round_trips_supported_keywords() {
let schema = round_trip::<JsonSchemaObject>(json!({
"type":"object",
"properties":{"name":{"type":"string"}},
"required":["name"],
"additionalProperties":false,
"$ref":"#/$defs/value",
"strict":true,
"extension":{"nested":[1,null]}
}));
assert_eq!(
schema.schema_type,
Some(JsonSchemaType::Name("object".into()))
);
assert_eq!(schema.required.as_deref(), Some(["name".into()].as_slice()));
assert_eq!(
schema.additional_properties.as_deref(),
Some(&JsonSchema::Boolean(false))
);
assert_eq!(schema.reference.as_deref(), Some("#/$defs/value"));
assert_eq!(schema.strict, Some(true));
}
#[rstest]
#[case::scalar(json!(7))]
#[case::properties_shape(json!({"properties":[]}))]
#[case::nested_schema(json!({"properties":{"name":7}}))]
#[case::schema_type(json!({"type":["string",7]}))]
#[case::items_shape(json!({"items":7}))]
#[case::tuple_member(json!({"items":[{"type":"string"},7]}))]
#[case::prefix_items_shape(json!({"prefixItems":{"type":"string"}}))]
#[case::composite_shape(json!({"anyOf":["string"]}))]
fn schemas_reject_malformed_known_keywords(#[case] wire: Value) {
assert!(serde_json::from_value::<JsonSchema>(wire).is_err());
}
#[rstest]
#[case::draft_07_tuple(json!({"type":"array","items":[{"type":"string"},true]}))]
#[case::draft_2020_12_tuple(json!({"type":"array","prefixItems":[{"type":"string"},true],"items":false}))]
fn tuple_schemas_expose_positional_members(#[case] wire: Value) {
let schema = round_trip::<JsonSchemaObject>(wire);
let members = match (&schema.items, &schema.prefix_items) {
(Some(JsonSchemaItems::Tuple(members)), None) => members,
(Some(JsonSchemaItems::Schema(rest)), Some(members)) => {
assert_eq!(rest.as_ref(), &JsonSchema::Boolean(false));
members
}
_ => panic!("expected positional members"),
};
let [JsonSchema::Object(first), JsonSchema::Boolean(true)] = members.as_slice() else {
panic!("expected string schema then true");
};
assert_eq!(
first.schema_type,
Some(JsonSchemaType::Name("string".into()))
);
assert!(schema.extra.is_empty());
}
#[rstest]
fn empty_schema_omits_null_optionals_and_preserves_arbitrary_keywords() {
let schema: JsonSchemaObject = serde_json::from_value(
json!({"type":null,"properties":null,"const":{"arbitrary":[1,null]},"future":null}),
)
.unwrap();
assert!(schema.schema_type.is_none());
assert!(schema.properties.is_none());
assert_eq!(
serde_json::to_value(schema).unwrap(),
json!({"const":{"arbitrary":[1,null]},"future":null})
);
}

9
litellm-rust/crates/llms-types/tests/messages_request.rs
View file

@ -33,6 +33,15 @@ fn content_block_type_schema_remains_a_string() {
#[case::compaction("compaction", ContentBlockType::Compaction)]
#[case::advisor_result("advisor_tool_result", ContentBlockType::AdvisorToolResult)]
#[case::web_search_result("web_search_tool_result", ContentBlockType::WebSearchToolResult)]
#[case::image("image", ContentBlockType::Other("image".into()))]
#[case::document("document", ContentBlockType::Other("document".into()))]
#[case::tool_addition("tool_addition", ContentBlockType::Other("tool_addition".into()))]
#[case::tool_removal("tool_removal", ContentBlockType::Other("tool_removal".into()))]
#[case::advisor("advisor_result", ContentBlockType::Other("advisor_result".into()))]
#[case::web_search_error(
"web_search_tool_result_error",
ContentBlockType::Other("web_search_tool_result_error".into())
)]
#[case::future_block("future_block", ContentBlockType::Other("future_block".into()))]
#[case::case_sensitive("Tool_Use", ContentBlockType::Other("Tool_Use".into()))]
#[case::empty("", ContentBlockType::Other(String::new()))]

1512
litellm-rust/crates/llms-types/tests/messages_types.rs Normal file
View file

File diff suppressed because it is too large Load diff

82
litellm-rust/crates/llms-types/tests/minimax.rs Normal file
View file

@ -0,0 +1,82 @@
use litellm_llms_types::providers::minimax::{
MinimaxMediaDetail, MinimaxMediaSource, MinimaxMessagesContentBlock,
};
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::image(json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA==","detail":"low"}}))]
#[case::video(json!({"type":"video","source":{"type":"url","url":"https://example.test/video","detail":"high","fps":1,"max_long_side_pixel":1024,"future":null},"cache_control":{"type":"ephemeral"}}))]
#[case::mid_conversation_system(json!({"type":"mid_conv_system","text":"instruction","future":null}))]
fn provider_content_blocks_round_trip(#[case] wire: Value) {
let block: MinimaxMessagesContentBlock = serde_json::from_value(wire.clone()).unwrap();
match &block {
MinimaxMessagesContentBlock::Image(image) => {
let MinimaxMediaSource::Base64 {
media_type,
data,
options,
} = &image.source
else {
panic!("expected base64 image source");
};
assert_eq!((media_type.as_str(), data.as_str()), ("image/png", "AA=="));
assert_eq!(options.detail, Some(MinimaxMediaDetail::Low));
assert!(options.extra.is_empty());
}
MinimaxMessagesContentBlock::Video(video) => {
let MinimaxMediaSource::Url { url, options } = &video.source else {
panic!("expected URL video source");
};
assert_eq!(url, "https://example.test/video");
assert_eq!(options.detail, Some(MinimaxMediaDetail::High));
assert_eq!(options.fps, Some(1.into()));
assert_eq!(options.max_long_side_pixel, Some(1024));
assert_eq!(options.extra["future"], Value::Null);
assert_eq!(
video.cache_control.as_ref().unwrap().cache_type.as_deref(),
Some("ephemeral")
);
}
MinimaxMessagesContentBlock::MidConvSystem { text, extra } => {
assert_eq!(text, "instruction");
assert_eq!(extra["future"], Value::Null);
}
}
assert_eq!(serde_json::to_value(block).unwrap(), wire);
}
#[rstest]
#[case::missing_source(json!({"type":"video"}))]
#[case::bad_source_tag(json!({"type":"image","source":{"type":"future"}}))]
#[case::url_without_url(json!({"type":"video","source":{"type":"url"}}))]
#[case::base64_without_data(json!({"type":"image","source":{"type":"base64","media_type":"image/png"}}))]
#[case::bad_detail(json!({"type":"image","source":{"type":"url","url":"u","detail":7}}))]
#[case::bad_fps(json!({"type":"video","source":{"type":"url","url":"u","fps":"fast"}}))]
#[case::missing_text(json!({"type":"mid_conv_system"}))]
fn provider_content_rejects_malformed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<MinimaxMessagesContentBlock>(wire).is_err());
}
#[rstest]
fn partial_media_options_omit_null_optionals() {
let block: MinimaxMessagesContentBlock = serde_json::from_value(json!({
"type":"video",
"source":{"type":"url","url":"u","detail":null,"fps":null,"future":null},
"cache_control":null
}))
.unwrap();
let MinimaxMessagesContentBlock::Video(video) = &block else {
panic!("expected video")
};
let MinimaxMediaSource::Url { options, .. } = &video.source else {
panic!("expected URL source");
};
assert!(options.detail.is_none());
assert!(options.fps.is_none());
assert!(video.cache_control.is_none());
assert_eq!(
serde_json::to_value(block).unwrap(),
json!({"type":"video","source":{"type":"url","url":"u","future":null}})
);
}

38
litellm-rust/crates/llms-types/tests/ocr.rs
View file

@ -1,4 +1,4 @@
use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrPage};
use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrBoundingBox, OcrDocument, OcrPage};
use rstest::rstest;
use serde_json::{Map, Value, json};
@ -102,3 +102,39 @@ fn response_serialization_preserves_extensions_and_native_presence(
assert_eq!(decoded.provider_native_response, native);
assert_eq!(decoded.into_json(), serialized);
}
#[rstest]
fn bounding_box_exposes_corner_coordinates_and_keeps_extensions() {
let wire = json!({"top_left_x":1,"top_left_y":2.5,"bottom_right_x":30,"bottom_right_y":40,"future":true});
let bounds: OcrBoundingBox = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(bounds.top_left_x, Some(1.into()));
assert_eq!(
bounds
.top_left_y
.as_ref()
.and_then(serde_json::Number::as_f64),
Some(2.5)
);
assert_eq!(bounds.bottom_right_x, Some(30.into()));
assert_eq!(bounds.bottom_right_y, Some(40.into()));
assert_eq!(Value::Object(bounds.extra.clone()), json!({"future":true}));
assert_eq!(serde_json::to_value(bounds).unwrap(), wire);
}
#[rstest]
fn partial_bounding_box_omits_null_corners() {
let bounds: OcrBoundingBox =
serde_json::from_value(json!({"top_left_x":null,"bottom_right_y":4})).unwrap();
assert!(bounds.top_left_x.is_none());
assert_eq!(
serde_json::to_value(bounds).unwrap(),
json!({"bottom_right_y":4})
);
}
#[rstest]
#[case::string_corner(json!({"top_left_x":"1"}))]
#[case::array_corner(json!({"bottom_right_y":[4]}))]
fn bounding_box_rejects_non_numeric_corners(#[case] wire: Value) {
assert!(serde_json::from_value::<OcrBoundingBox>(wire).is_err());
}

671
litellm-rust/crates/llms-types/tests/responses.rs
View file

@ -1,5 +1,31 @@
use litellm_llms_types::formats::responses::streaming_websocket::ResponsesWsEventType;
use litellm_llms_types::formats::chat_completions::{PromptCacheBreakpoint, PromptCacheMode};
use litellm_llms_types::formats::responses::{
ResponsesAdditionalTools, ResponsesApplyPatchCall, ResponsesApplyPatchCallOutput,
ResponsesApplyPatchOperation, ResponsesCodeOutput, ResponsesCompaction,
ResponsesComputerAction, ResponsesComputerCall, ResponsesComputerCallOutput,
ResponsesComputerOutput, ResponsesContentPart, ResponsesCoordinate, ResponsesCustomToolCall,
ResponsesCustomToolCallOutput, ResponsesFunctionCall, ResponsesFunctionCallOutput,
ResponsesImageGenerationCall, ResponsesInputContent, ResponsesLocalShellAction,
ResponsesLocalShellCall, ResponsesLocalShellCallOutput, ResponsesMcpApprovalRequest,
ResponsesMcpApprovalResponse, ResponsesMcpError, ResponsesMcpErrorDetail, ResponsesOutputItem,
ResponsesProgram, ResponsesProgramOutput, ResponsesSafetyCheck, ResponsesShellAction,
ResponsesShellCall, ResponsesShellCallOutput, ResponsesShellEnvironment, ResponsesShellOutcome,
ResponsesShellOutputChunk, ResponsesToolCaller, ResponsesToolOutput, ResponsesToolSearchCall,
ResponsesToolSearchOutput, ResponsesWebSearchAction,
streaming_websocket::{ResponsesEventResponse, ResponsesWsEventType},
};
use litellm_llms_types::recognized::Recognized;
use rstest::rstest;
use serde::{Serialize, de::DeserializeOwned};
use serde_json::{Value, json};
fn round_trip<T>(wire: Value)
where
T: DeserializeOwned + Serialize,
{
let parsed: T = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(parsed).unwrap(), wire);
}
#[rstest]
#[case::create("response.create", ResponsesWsEventType::ResponseCreate)]
@ -46,3 +72,646 @@ fn websocket_event_type_schema_is_open_string() {
Some(&serde_json::json!("string"))
);
}
#[rstest]
#[case::message(json!({"type":"message","role":"assistant","content":[{"type":"output_text","text":"answer","annotations":[{"type":"url_citation","url":"https://example.test","start_index":0,"end_index":6}]}]}))]
#[case::function_call(json!({"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{\"query\":\"q\"}"}))]
#[case::custom_tool(json!({"type":"custom_tool_call","name":"lookup","input":"q"}))]
#[case::reasoning(json!({"type":"reasoning","summary":[{"type":"summary_text","text":"summary"}],"encrypted_content":"opaque"}))]
#[case::web_search(json!({"type":"web_search_call","action":{"type":"search","queries":["q"],"sources":[{"type":"url","url":"https://example.test"}]}}))]
#[case::file_search(json!({"type":"file_search_call","queries":["q"],"results":[{"file_id":"file_1","score":1,"attributes":{"custom":[1,null]}}]}))]
#[case::code(json!({"type":"code_interpreter_call","outputs":[{"type":"logs","logs":"done"},{"type":"image","url":"https://example.test"}]}))]
#[case::image(json!({"type":"image_generation_call","result":"generated"}))]
#[case::mcp(json!({"type":"mcp_call","server_label":"server","name":"lookup","arguments":"{}","output":"done"}))]
fn output_items_round_trip(#[case] wire: Value) {
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
match &item {
ResponsesOutputItem::Message(message) => {
assert_eq!(message.role.as_deref(), Some("assistant"));
let Some(content) = &message.content else {
panic!("expected content")
};
let [
ResponsesContentPart::OutputText {
text,
annotations: Some(annotations),
..
},
] = content.as_slice()
else {
panic!("expected output text and annotations")
};
assert_eq!(text, "answer");
let [
litellm_llms_types::formats::responses::ResponsesAnnotation::UrlCitation(citation),
] = annotations.as_slice()
else {
panic!("expected URL citation")
};
assert_eq!(citation.url.as_deref(), Some("https://example.test"));
assert_eq!(citation.start_index, Some(0));
}
ResponsesOutputItem::FunctionCall(call) => {
assert_eq!(call.call_id.as_deref(), Some("call_1"));
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{\"query\":\"q\"}"));
}
ResponsesOutputItem::CustomToolCall(call) => {
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.input.as_deref(), Some("q"));
}
ResponsesOutputItem::Reasoning(reasoning) => {
assert_eq!(reasoning.encrypted_content.as_deref(), Some("opaque"));
let Some(summary) = &reasoning.summary else {
panic!("expected summary")
};
let [ResponsesContentPart::SummaryText { text, .. }] = summary.as_slice() else {
panic!("expected summary text")
};
assert_eq!(text, "summary");
}
ResponsesOutputItem::WebSearchCall(call) => {
let Some(ResponsesWebSearchAction::Search {
queries: Some(queries),
sources: Some(sources),
..
}) = &call.action
else {
panic!("expected search action")
};
assert_eq!(queries, &["q"]);
assert_eq!(sources[0].url.as_deref(), Some("https://example.test"));
}
ResponsesOutputItem::FileSearchCall(call) => {
assert_eq!(call.queries.as_deref(), Some(["q".to_owned()].as_slice()));
let Some(results) = &call.results else {
panic!("expected search results")
};
assert_eq!(results[0].file_id.as_deref(), Some("file_1"));
assert_eq!(results[0].score, Some(1.into()));
assert_eq!(
results[0].attributes.as_ref().unwrap()["custom"],
json!([1, null])
);
}
ResponsesOutputItem::CodeInterpreterCall(call) => {
let Some(outputs) = &call.outputs else {
panic!("expected code outputs")
};
let [
ResponsesCodeOutput::Logs { logs, .. },
ResponsesCodeOutput::Image { url, .. },
] = outputs.as_slice()
else {
panic!("expected logs and image")
};
assert_eq!(logs, "done");
assert_eq!(url, "https://example.test");
}
ResponsesOutputItem::ImageGenerationCall(call) => {
assert_eq!(call.result.as_deref(), Some("generated"))
}
ResponsesOutputItem::McpCall(call) => {
assert_eq!(call.server_label.as_deref(), Some("server"));
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{}"));
assert_eq!(call.output.as_deref(), Some("done"));
}
other => panic!("unexpected variant {other:?}"),
}
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
fn nested_event_response_exposes_typed_output_and_preserves_extensions() {
let wire = json!({
"id":"response_1",
"model":"example-model",
"status":"completed",
"output":[{"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{}","extension":true}],
"extension":{"nested":[1,null]}
});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(response.id.as_deref(), Some("response_1"));
assert_eq!(response.model.as_deref(), Some("example-model"));
let Some(output) = &response.output else {
panic!("expected typed output");
};
let [Recognized::Known(ResponsesOutputItem::FunctionCall(call))] = output.as_slice() else {
panic!("expected function call");
};
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(call.arguments.as_deref(), Some("{}"));
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
#[rstest]
#[case::empty(json!({}))]
#[case::partial(json!({"id":"response_1","output":[]}))]
fn nested_event_response_accepts_partial_metadata(#[case] wire: Value) {
round_trip::<ResponsesEventResponse>(wire);
}
#[rstest]
#[case::wrong_model(json!({"model":7}))]
#[case::wrong_status(json!({"status":false}))]
#[case::wrong_output(json!({"output":{}}))]
fn event_response_rejects_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesEventResponse>(wire).is_err());
}
#[rstest]
#[case::unknown_type(json!({"type":"future_item","id":"item_1","payload":[1,null]}))]
#[case::missing_tag(json!({"id":"item_1"}))]
#[case::malformed_known(json!({"type":"message","content":[{"type":"output_text","text":7}]}))]
fn event_response_keeps_unmodeled_output_items_beside_typed_ones(#[case] item: Value) {
let wire = json!({"output":[{"type":"function_call","name":"lookup"}, item.clone()]});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
let Some(
[
Recognized::Known(ResponsesOutputItem::FunctionCall(call)),
Recognized::Unrecognized(kept),
],
) = response.output.as_deref()
else {
panic!("expected one typed item and one preserved item");
};
assert_eq!(call.name.as_deref(), Some("lookup"));
assert_eq!(kept, &item);
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
#[rstest]
#[case::find_in_page(
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}),
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"})
)]
#[case::legacy_find(
json!({"type":"find","url":"https://example.test","pattern":"needle"}),
json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"})
)]
fn web_search_find_action_exposes_url_and_pattern(#[case] wire: Value, #[case] serialized: Value) {
let action: ResponsesWebSearchAction = serde_json::from_value(wire).unwrap();
let ResponsesWebSearchAction::FindInPage {
url,
pattern,
extra,
} = &action
else {
panic!("expected find_in_page action");
};
assert_eq!(url, "https://example.test");
assert_eq!(pattern, "needle");
assert!(extra.is_empty());
assert_eq!(serde_json::to_value(action).unwrap(), serialized);
}
#[rstest]
#[case::with_url(json!({"type":"open_page","url":"https://example.test"}), Some("https://example.test"))]
#[case::without_url(json!({"type":"open_page"}), None)]
fn web_search_open_page_url_is_optional(#[case] wire: Value, #[case] expected: Option<&str>) {
let action: ResponsesWebSearchAction = serde_json::from_value(wire.clone()).unwrap();
let ResponsesWebSearchAction::OpenPage { url, .. } = &action else {
panic!("expected open_page action");
};
assert_eq!(url.as_deref(), expected);
assert_eq!(serde_json::to_value(action).unwrap(), wire);
}
#[rstest]
#[case::find_pattern(json!({"type":"find_in_page","url":"u","pattern":false}))]
#[case::find_missing_url(json!({"type":"find_in_page","pattern":"p"}))]
#[case::open_page_url(json!({"type":"open_page","url":7}))]
fn web_search_action_rejects_malformed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesWebSearchAction>(wire).is_err());
}
#[rstest]
fn event_response_optional_fields_omit_missing_and_null() {
let response: ResponsesEventResponse = serde_json::from_value(json!({
"id":null,"model":null,"status":null,"output":null,"future":null
}))
.unwrap();
assert!(response.id.is_none());
assert!(response.model.is_none());
assert!(response.status.is_none());
assert!(response.output.is_none());
assert_eq!(
serde_json::to_value(response).unwrap(),
json!({"future":null})
);
}
#[rstest]
#[case::refusal(json!({"type":"refusal","refusal":"refused","future":null}), ResponsesContentPart::Refusal { refusal:"refused".into(), extra:serde_json::Map::from_iter([("future".into(), Value::Null)]) })]
#[case::reasoning(json!({"type":"reasoning_text","text":"reason"}), ResponsesContentPart::ReasoningText { text:"reason".into(), extra:Default::default() })]
fn content_parts_expose_typed_variants(
#[case] wire: Value,
#[case] expected: ResponsesContentPart,
) {
let content: ResponsesContentPart = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(content, expected);
assert_eq!(serde_json::to_value(content).unwrap(), wire);
}
#[rstest]
fn mcp_discovery_exposes_tools_and_nested_schemas() {
let wire = json!({
"type":"mcp_list_tools","id":"item_1","server_label":"tools",
"tools":[{"name":"lookup","description":"Look up a value","input_schema":{"type":"object","properties":{"query":{"type":"string"}}},"annotations":{"read_only":false},"future":null}],
"extension":[1,null]
});
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
let ResponsesOutputItem::McpListTools(discovery) = &item else {
panic!("expected discovery")
};
assert_eq!(discovery.server_label.as_deref(), Some("tools"));
let tool = &discovery.tools.as_ref().unwrap()[0];
assert_eq!(tool.name.as_deref(), Some("lookup"));
let Some(litellm_llms_types::json_schema::JsonSchema::Object(schema)) = &tool.input_schema
else {
panic!("expected tool schema")
};
assert!(schema.properties.as_ref().unwrap().contains_key("query"));
assert_eq!(tool.annotations, Some(json!({"read_only":false})));
assert_eq!(tool.extra["future"], Value::Null);
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
#[case::partial(json!({"server_label":"tools","tools":[]}))]
#[case::failure(json!({"server_label":"tools","error":"unavailable"}))]
fn mcp_discovery_accepts_partial_payloads(#[case] wire: Value) {
let discovery: litellm_llms_types::formats::responses::ResponsesMcpListTools =
serde_json::from_value(wire.clone()).unwrap();
assert_eq!(discovery.server_label.as_deref(), Some("tools"));
assert!(discovery.id.is_none());
assert_eq!(serde_json::to_value(discovery).unwrap(), wire);
}
#[rstest]
#[case::message(json!("unavailable"), ResponsesMcpError::Message("unavailable".into()))]
#[case::protocol(json!({"type":"mcp_protocol_error","code":-32600,"message":"invalid","extension":null}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpProtocolError {code:-32600,message:"invalid".into(),extra:serde_json::Map::from_iter([("extension".into(), Value::Null)])}))]
#[case::http(json!({"type":"http_error","code":503,"message":"unavailable"}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::HttpError {code:503,message:"unavailable".into(),extra:Default::default()}))]
#[case::tool(json!({"type":"mcp_tool_execution_error","content":{"arbitrary":[1,null]}}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpToolExecutionError {content:json!({"arbitrary":[1,null]}),extra:Default::default()}))]
fn mcp_call_errors_expose_typed_variants(#[case] wire: Value, #[case] expected: ResponsesMcpError) {
let error: ResponsesMcpError = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(error, expected);
assert_eq!(serde_json::to_value(&error).unwrap(), wire);
let call: ResponsesOutputItem =
serde_json::from_value(json!({"type":"mcp_call","error":wire})).unwrap();
let ResponsesOutputItem::McpCall(call) = call else {
panic!("expected call")
};
assert_eq!(call.error, Some(error));
}
#[rstest]
#[case::tools_not_array(json!({"type":"mcp_list_tools","tools":{}}))]
#[case::invalid_name(json!({"type":"mcp_list_tools","tools":[{"name":7}]}))]
#[case::invalid_schema(json!({"type":"mcp_list_tools","tools":[{"input_schema":{"properties":{"query":7}}}]}))]
#[case::invalid_error_code(json!({"type":"mcp_call","error":{"type":"http_error","code":"503","message":"unavailable"}}))]
#[case::missing_error_content(json!({"type":"mcp_call","error":{"type":"mcp_tool_execution_error"}}))]
#[case::unknown_error_tag(json!({"type":"mcp_call","error":{"type":"future"}}))]
fn mcp_output_rejects_malformed_known_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesOutputItem>(wire).is_err());
}
fn text(value: &str) -> Option<String> {
Some(value.to_owned())
}
fn program_caller() -> Option<ResponsesToolCaller> {
Some(ResponsesToolCaller::Program {
caller_id: "prog_1".into(),
extra: Default::default(),
})
}
fn direct_caller() -> Option<ResponsesToolCaller> {
Some(ResponsesToolCaller::Direct {
extra: Default::default(),
})
}
fn safety_check() -> ResponsesSafetyCheck {
ResponsesSafetyCheck {
id: text("sc_1"),
code: text("malicious_instructions"),
message: text("check"),
..Default::default()
}
}
#[rstest]
#[case::function_call(
json!({"type":"function_call","id":"fc_1","status":"completed","call_id":"call_1","name":"lookup","arguments":"{}","namespace":"ns","async":true,"caller":{"type":"program","caller_id":"prog_1"}}),
ResponsesOutputItem::FunctionCall(ResponsesFunctionCall {
id: text("fc_1"), status: text("completed"), call_id: text("call_1"), name: text("lookup"),
arguments: text("{}"), namespace: text("ns"), r#async: Some(true), caller: program_caller(), ..Default::default()
})
)]
#[case::custom_tool_call(
json!({"type":"custom_tool_call","call_id":"call_1","name":"lookup","input":"q","namespace":"ns","async":false,"caller":{"type":"direct"}}),
ResponsesOutputItem::CustomToolCall(ResponsesCustomToolCall {
call_id: text("call_1"), name: text("lookup"), input: text("q"), namespace: text("ns"),
r#async: Some(false), caller: direct_caller(), ..Default::default()
})
)]
#[case::image_generation_call(
json!({"type":"image_generation_call","id":"ig_1","status":"completed","result":"b64","action":"edit","background":"opaque","output_format":"webp","quality":"high","revised_prompt":"a cat","size":"1536x864"}),
ResponsesOutputItem::ImageGenerationCall(ResponsesImageGenerationCall {
id: text("ig_1"), status: text("completed"), result: text("b64"), action: text("edit"), background: text("opaque"),
output_format: text("webp"), quality: text("high"), revised_prompt: text("a cat"), size: text("1536x864"), ..Default::default()
})
)]
#[case::function_call_output_text(
json!({"type":"function_call_output","id":"fco_1","status":"completed","call_id":"call_1","output":"done","caller":{"type":"direct"},"created_by":"user_1","name":"lookup","namespace":"ns"}),
ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput {
id: text("fco_1"), status: text("completed"), call_id: text("call_1"), output: Some(ResponsesToolOutput::Text("done".into())),
caller: direct_caller(), created_by: text("user_1"), name: text("lookup"), namespace: text("ns"), ..Default::default()
})
)]
#[case::function_call_output_content(
json!({"type":"function_call_output","call_id":"call_1","output":[
{"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"explicit"}},
{"type":"input_image","detail":"low","file_id":"file_1","image_url":"https://example.test/i.png"},
{"type":"input_file","detail":"high","file_data":"data","file_id":"file_2","file_url":"https://example.test/f","filename":"f.pdf"}
]}),
ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput {
call_id: text("call_1"),
output: Some(ResponsesToolOutput::Content(vec![
ResponsesInputContent::InputText {
text: "t".into(),
prompt_cache_breakpoint: Some(PromptCacheBreakpoint { mode: PromptCacheMode::Explicit, extra: Default::default() }),
extra: Default::default(),
},
ResponsesInputContent::InputImage {
detail: "low".into(), file_id: text("file_1"), image_url: text("https://example.test/i.png"),
prompt_cache_breakpoint: None, extra: Default::default(),
},
ResponsesInputContent::InputFile {
detail: text("high"), file_data: text("data"), file_id: text("file_2"), file_url: text("https://example.test/f"),
filename: text("f.pdf"), prompt_cache_breakpoint: None, extra: Default::default(),
},
])),
..Default::default()
})
)]
#[case::custom_tool_call_output(
json!({"type":"custom_tool_call_output","id":"cto_1","status":"completed","call_id":"call_1","output":[{"type":"input_text","text":"t"}],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}),
ResponsesOutputItem::CustomToolCallOutput(ResponsesCustomToolCallOutput {
id: text("cto_1"), status: text("completed"), call_id: text("call_1"),
output: Some(ResponsesToolOutput::Content(vec![ResponsesInputContent::InputText {
text: "t".into(), prompt_cache_breakpoint: None, extra: Default::default(),
}])),
caller: program_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::computer_call(
json!({"type":"computer_call","id":"cu_1","status":"completed","call_id":"call_1",
"pending_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}],
"action":{"type":"click","button":"left","x":1,"y":2,"keys":["shift"]},
"actions":[
{"type":"double_click","x":3,"y":4},
{"type":"drag","path":[{"x":5,"y":6},{"x":7,"y":8}],"keys":["ctrl"]},
{"type":"keypress","keys":["enter"]},
{"type":"move","x":-1,"y":9},
{"type":"screenshot"},
{"type":"scroll","scroll_x":0,"scroll_y":-10,"x":11,"y":12},
{"type":"type","text":"hello"},
{"type":"wait"}
]}),
ResponsesOutputItem::ComputerCall(ResponsesComputerCall {
id: text("cu_1"), status: text("completed"), call_id: text("call_1"), pending_safety_checks: Some(vec![safety_check()]),
action: Some(ResponsesComputerAction::Click { button: "left".into(), x: 1, y: 2, keys: Some(vec!["shift".into()]), extra: Default::default() }),
actions: Some(vec![
ResponsesComputerAction::DoubleClick { x: 3, y: 4, keys: None, extra: Default::default() },
ResponsesComputerAction::Drag {
path: vec![
ResponsesCoordinate { x: 5, y: 6, extra: Default::default() },
ResponsesCoordinate { x: 7, y: 8, extra: Default::default() },
],
keys: Some(vec!["ctrl".into()]),
extra: Default::default(),
},
ResponsesComputerAction::Keypress { keys: vec!["enter".into()], extra: Default::default() },
ResponsesComputerAction::Move { x: -1, y: 9, keys: None, extra: Default::default() },
ResponsesComputerAction::Screenshot { extra: Default::default() },
ResponsesComputerAction::Scroll { scroll_x: 0, scroll_y: -10, x: 11, y: 12, keys: None, extra: Default::default() },
ResponsesComputerAction::Type { text: "hello".into(), extra: Default::default() },
ResponsesComputerAction::Wait { extra: Default::default() },
]),
..Default::default()
})
)]
#[case::computer_call_output(
json!({"type":"computer_call_output","id":"cuo_1","status":"completed","call_id":"call_1","output":{"type":"computer_screenshot","file_id":"file_1","image_url":"https://example.test/s.png"},"acknowledged_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}],"created_by":"user_1"}),
ResponsesOutputItem::ComputerCallOutput(ResponsesComputerCallOutput {
id: text("cuo_1"), status: text("completed"), call_id: text("call_1"),
output: Some(ResponsesComputerOutput::ComputerScreenshot { file_id: text("file_1"), image_url: text("https://example.test/s.png"), extra: Default::default() }),
acknowledged_safety_checks: Some(vec![safety_check()]), created_by: text("user_1"), ..Default::default()
})
)]
#[case::program(
json!({"type":"program","id":"prog_item","call_id":"prog_1","code":"run()","fingerprint":"fp"}),
ResponsesOutputItem::Program(ResponsesProgram {
id: text("prog_item"), call_id: text("prog_1"), code: text("run()"), fingerprint: text("fp"), ..Default::default()
})
)]
#[case::program_output(
json!({"type":"program_output","id":"po_1","status":"completed","call_id":"prog_1","result":"42"}),
ResponsesOutputItem::ProgramOutput(ResponsesProgramOutput {
id: text("po_1"), status: text("completed"), call_id: text("prog_1"), result: text("42"), ..Default::default()
})
)]
#[case::tool_search_call(
json!({"type":"tool_search_call","id":"ts_1","status":"completed","call_id":"call_1","arguments":{"query":["weather",null]},"execution":"server","created_by":"user_1"}),
ResponsesOutputItem::ToolSearchCall(ResponsesToolSearchCall {
id: text("ts_1"), status: text("completed"), call_id: text("call_1"), arguments: Some(json!({"query":["weather",null]})),
execution: text("server"), created_by: text("user_1"), ..Default::default()
})
)]
#[case::tool_search_output(
json!({"type":"tool_search_output","id":"tso_1","status":"completed","call_id":"call_1","execution":"client","tools":[{"type":"function","name":"lookup","parameters":null,"strict":true}],"created_by":"user_1"}),
ResponsesOutputItem::ToolSearchOutput(ResponsesToolSearchOutput {
id: text("tso_1"), status: text("completed"), call_id: text("call_1"), execution: text("client"),
tools: Some(vec![serde_json::Map::from_iter([
("type".into(), json!("function")), ("name".into(), json!("lookup")),
("parameters".into(), Value::Null), ("strict".into(), json!(true)),
])]),
created_by: text("user_1"), ..Default::default()
})
)]
#[case::additional_tools(
json!({"type":"additional_tools","id":"at_1","role":"developer","tools":[{"type":"local_shell"}]}),
ResponsesOutputItem::AdditionalTools(ResponsesAdditionalTools {
id: text("at_1"), role: text("developer"),
tools: Some(vec![serde_json::Map::from_iter([("type".into(), json!("local_shell"))])]), ..Default::default()
})
)]
#[case::compaction(
json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque","created_by":"user_1"}),
ResponsesOutputItem::Compaction(ResponsesCompaction {
id: text("cmp_1"), encrypted_content: text("opaque"), created_by: text("user_1"), ..Default::default()
})
)]
#[case::local_shell_call(
json!({"type":"local_shell_call","id":"ls_1","status":"completed","call_id":"call_1","action":{"type":"exec","command":["ls","-a"],"env":{"HOME":"/home/u"},"timeout_ms":1000,"user":"u","working_directory":"/tmp"}}),
ResponsesOutputItem::LocalShellCall(ResponsesLocalShellCall {
id: text("ls_1"), status: text("completed"), call_id: text("call_1"),
action: Some(ResponsesLocalShellAction::Exec {
command: vec!["ls".into(), "-a".into()], env: [("HOME".into(), "/home/u".into())].into(),
timeout_ms: Some(1000), user: text("u"), working_directory: text("/tmp"), extra: Default::default(),
}),
..Default::default()
})
)]
#[case::local_shell_call_output(
json!({"type":"local_shell_call_output","id":"lso_1","status":"completed","output":"{\"stdout\":\"x\"}"}),
ResponsesOutputItem::LocalShellCallOutput(ResponsesLocalShellCallOutput {
id: text("lso_1"), status: text("completed"), output: text("{\"stdout\":\"x\"}"), ..Default::default()
})
)]
#[case::shell_call_local(
json!({"type":"shell_call","id":"sh_1","status":"in_progress","call_id":"call_1","action":{"commands":["ls"],"max_output_length":100,"timeout_ms":500},"environment":{"type":"local"},"caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ShellCall(ResponsesShellCall {
id: text("sh_1"), status: text("in_progress"), call_id: text("call_1"),
action: Some(ResponsesShellAction { commands: Some(vec!["ls".into()]), max_output_length: Some(100), timeout_ms: Some(500), ..Default::default() }),
environment: Some(ResponsesShellEnvironment::Local { extra: Default::default() }),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::shell_call_container(
json!({"type":"shell_call","call_id":"call_1","environment":{"type":"container_reference","container_id":"cntr_1"}}),
ResponsesOutputItem::ShellCall(ResponsesShellCall {
call_id: text("call_1"),
environment: Some(ResponsesShellEnvironment::ContainerReference { container_id: "cntr_1".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::shell_call_output(
json!({"type":"shell_call_output","id":"sho_1","status":"completed","call_id":"call_1","max_output_length":100,"output":[
{"outcome":{"type":"exit","exit_code":2},"stdout":"out","stderr":"err","created_by":"user_1"},
{"outcome":{"type":"timeout"},"stdout":"","stderr":""}
],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}),
ResponsesOutputItem::ShellCallOutput(ResponsesShellCallOutput {
id: text("sho_1"), status: text("completed"), call_id: text("call_1"), max_output_length: Some(100),
output: Some(vec![
ResponsesShellOutputChunk {
outcome: Some(ResponsesShellOutcome::Exit { exit_code: 2, extra: Default::default() }),
stdout: text("out"), stderr: text("err"), created_by: text("user_1"), ..Default::default()
},
ResponsesShellOutputChunk {
outcome: Some(ResponsesShellOutcome::Timeout { extra: Default::default() }),
stdout: text(""), stderr: text(""), ..Default::default()
},
]),
caller: program_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::apply_patch_create(
json!({"type":"apply_patch_call","id":"ap_1","status":"completed","call_id":"call_1","operation":{"type":"create_file","path":"a.txt","diff":"+a"},"caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
id: text("ap_1"), status: text("completed"), call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::CreateFile { path: "a.txt".into(), diff: "+a".into(), extra: Default::default() }),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::apply_patch_delete(
json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"delete_file","path":"a.txt"}}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::DeleteFile { path: "a.txt".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::apply_patch_update(
json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"update_file","path":"a.txt","diff":"-a\n+b"}}),
ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall {
call_id: text("call_1"),
operation: Some(ResponsesApplyPatchOperation::UpdateFile { path: "a.txt".into(), diff: "-a\n+b".into(), extra: Default::default() }),
..Default::default()
})
)]
#[case::apply_patch_call_output(
json!({"type":"apply_patch_call_output","id":"apo_1","status":"failed","call_id":"call_1","output":"conflict","caller":{"type":"direct"},"created_by":"user_1"}),
ResponsesOutputItem::ApplyPatchCallOutput(ResponsesApplyPatchCallOutput {
id: text("apo_1"), status: text("failed"), call_id: text("call_1"), output: text("conflict"),
caller: direct_caller(), created_by: text("user_1"), ..Default::default()
})
)]
#[case::mcp_approval_request(
json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}),
ResponsesOutputItem::McpApprovalRequest(ResponsesMcpApprovalRequest {
id: text("apr_1"), server_label: text("s"), name: text("n"), arguments: text("{}"), ..Default::default()
})
)]
#[case::mcp_approval_response(
json!({"type":"mcp_approval_response","id":"aprr_1","approval_request_id":"apr_1","approve":false,"reason":"denied"}),
ResponsesOutputItem::McpApprovalResponse(ResponsesMcpApprovalResponse {
id: text("aprr_1"), approval_request_id: text("apr_1"), approve: Some(false), reason: text("denied"), ..Default::default()
})
)]
fn documented_output_items_parse_into_typed_variants(
#[case] wire: Value,
#[case] expected: ResponsesOutputItem,
) {
let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(item, expected);
assert_eq!(serde_json::to_value(item).unwrap(), wire);
}
#[rstest]
#[case::compaction(json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque"}))]
#[case::approval(json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}))]
#[case::shell(json!({"type":"shell_call","call_id":"call_1","action":{"commands":["ls"]}}))]
fn event_response_types_documented_output_items(#[case] item: Value) {
let wire = json!({"output":[item.clone()]});
let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap();
let Some([Recognized::Known(known)]) = response.output.as_deref() else {
panic!("expected one typed item");
};
assert_eq!(
known,
&serde_json::from_value::<ResponsesOutputItem>(item).unwrap()
);
assert_eq!(serde_json::to_value(response).unwrap(), wire);
}
#[rstest]
#[case::unknown_caller(json!({"type":"function_call","caller":{"type":"future"}}))]
#[case::program_caller_missing_id(json!({"type":"shell_call","caller":{"type":"program"}}))]
#[case::untagged_caller(json!({"type":"custom_tool_call","caller":{}}))]
#[case::async_not_bool(json!({"type":"function_call","async":"yes"}))]
#[case::output_not_text_or_list(json!({"type":"function_call_output","output":{"text":"t"}}))]
#[case::unknown_input_content(json!({"type":"function_call_output","output":[{"type":"input_audio"}]}))]
#[case::input_text_missing_text(json!({"type":"custom_tool_call_output","output":[{"type":"input_text"}]}))]
#[case::input_image_missing_detail(json!({"type":"custom_tool_call_output","output":[{"type":"input_image","file_id":"f"}]}))]
#[case::bad_cache_mode(json!({"type":"custom_tool_call_output","output":[{"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"implicit"}}]}))]
#[case::unknown_computer_action(json!({"type":"computer_call","action":{"type":"teleport"}}))]
#[case::click_missing_button(json!({"type":"computer_call","action":{"type":"click","x":1,"y":2}}))]
#[case::click_fractional_x(json!({"type":"computer_call","action":{"type":"click","button":"left","x":1.5,"y":2}}))]
#[case::drag_point_missing_y(json!({"type":"computer_call","actions":[{"type":"drag","path":[{"x":1}]}]}))]
#[case::keypress_keys_not_list(json!({"type":"computer_call","actions":[{"type":"keypress","keys":"enter"}]}))]
#[case::safety_check_not_object(json!({"type":"computer_call","pending_safety_checks":["sc_1"]}))]
#[case::unknown_screenshot_tag(json!({"type":"computer_call_output","output":{"type":"screenshot"}}))]
#[case::untagged_screenshot(json!({"type":"computer_call_output","output":{"file_id":"f"}}))]
#[case::tool_not_object(json!({"type":"tool_search_output","tools":["lookup"]}))]
#[case::unknown_local_shell_action(json!({"type":"local_shell_call","action":{"type":"spawn","command":[],"env":{}}}))]
#[case::local_shell_missing_env(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"]}}))]
#[case::local_shell_env_value(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"],"env":{"A":1}}}))]
#[case::shell_commands_not_list(json!({"type":"shell_call","action":{"commands":"ls"}}))]
#[case::unknown_shell_environment(json!({"type":"shell_call","environment":{"type":"container_auto"}}))]
#[case::container_missing_id(json!({"type":"shell_call","environment":{"type":"container_reference"}}))]
#[case::unknown_shell_outcome(json!({"type":"shell_call_output","output":[{"outcome":{"type":"killed"}}]}))]
#[case::exit_missing_code(json!({"type":"shell_call_output","output":[{"outcome":{"type":"exit"}}]}))]
#[case::negative_max_output(json!({"type":"shell_call_output","max_output_length":-1}))]
#[case::unknown_patch_operation(json!({"type":"apply_patch_call","operation":{"type":"rename_file","path":"a"}}))]
#[case::update_missing_diff(json!({"type":"apply_patch_call","operation":{"type":"update_file","path":"a"}}))]
#[case::approve_not_bool(json!({"type":"mcp_approval_response","approve":"true"}))]
#[case::compaction_content_not_string(json!({"type":"compaction","encrypted_content":7}))]
#[case::program_code_not_string(json!({"type":"program","code":["run()"]}))]
fn output_items_reject_malformed_typed_fields(#[case] wire: Value) {
assert!(serde_json::from_value::<ResponsesOutputItem>(wire).is_err());
}

36
litellm-rust/crates/llms/src/anthropic/beta_headers.rs Normal file
View file

@ -0,0 +1,36 @@
use litellm_http::request::{with_header, without_headers};
use litellm_llms_types::providers::anthropic::{BetaProvider, BetaSet};
use crate::{anthropic::common_utils::existing_betas, base_llm::auth::Headers};
const BETA_HEADER: &str = "anthropic-beta";
/// How the `anthropic-beta` header is treated before a request leaves for a host.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum BetaPolicy {
/// Send the header untouched and let the host decide.
Forward,
/// Keep only the betas the provider accepts, under the names it expects.
Filter(BetaProvider),
/// Remove every `anthropic-beta` header.
Drop,
}
impl BetaPolicy {
pub fn apply(self, headers: Headers) -> Headers {
let provider = match self {
Self::Forward => return headers,
Self::Drop => return without_headers(headers, &[BETA_HEADER]),
Self::Filter(provider) => provider,
};
let accepted: BetaSet = existing_betas(&headers)
.iter()
.filter_map(|beta| beta.on(provider))
.collect();
let headers = without_headers(headers, &[BETA_HEADER]);
if accepted.is_empty() {
return headers;
}
with_header(headers, BETA_HEADER, accepted.to_string())
}
}

2
litellm-rust/crates/llms/src/anthropic/chat/transformation.rs
View file

@ -241,7 +241,7 @@ fn anthropic_body(
.iter()
.map(|turn| {
json!({
"role": turn.role.as_str(),
"role": <&'static str>::from(turn.role),
"content": turn.texts.iter().map(|text| text_block(text)).collect::<Vec<_>>(),
})
})

14
litellm-rust/crates/llms/src/anthropic/common_utils.rs
View file

@ -596,17 +596,10 @@ mod tests {
use crate::base_llm::messages::context::SupportedEffortTiers;
use rstest::{fixture, rstest};
use serde_json::json;
use strum::VariantArray;
use super::*;
const ALL_LEVELS: [EffortLevel; 5] = [
EffortLevel::Low,
EffortLevel::Medium,
EffortLevel::High,
EffortLevel::Xhigh,
EffortLevel::Max,
];
fn apply(sanitizer: fn(Vec<Message>) -> Vec<Message>, messages: Value) -> Value {
let parsed: Vec<Message> = serde_json::from_value(messages).unwrap();
serde_json::to_value(sanitizer(parsed)).unwrap()
@ -1712,7 +1705,10 @@ mod tests {
..unmapped
};
assert_eq!(
ALL_LEVELS.map(|level| supports_effort_tier(&capabilities, level)),
EffortLevel::VARIANTS
.iter()
.map(|level| supports_effort_tier(&capabilities, *level))
.collect::<Vec<_>>(),
expected
);
}

10
litellm-rust/crates/llms/src/anthropic/messages/thinking.rs
View file

@ -10,6 +10,7 @@ use litellm_llms_types::{
};
use litellm_python_compat::{json::from_json, repr::repr, truthy::truthy};
use serde_json::Value;
use strum::VariantArray;
use crate::base_llm::messages::context::{
MessagesModelCapabilities, ThinkingBudgets, ThinkingContext,
@ -54,14 +55,17 @@ fn unmapped_effort(effort: &Value) -> Error {
Error::InvalidRequest(crate::ErrorDetail::InvalidChoice {
field: "reasoning effort",
actual: repr(&from_json(effort.clone())),
choices: ReasoningEffort::ALL.map(|effort| effort.as_str()).into(),
choices: ReasoningEffort::VARIANTS
.iter()
.map(|effort| <&'static str>::from(*effort))
.collect(),
})
}
fn unsupported_effort(level: EffortLevel, model: &str) -> Error {
Error::InvalidRequest(crate::ErrorDetail::UnsupportedValue {
field: "effort",
value: level.as_str(),
value: <&'static str>::from(level),
model: model.into(),
})
}
@ -134,7 +138,7 @@ fn legacy_reasoning_effort(
Some(Recognized::Known(level)) => Ok((*level).into()),
Some(Recognized::Unrecognized(value)) if truthy(&from_json(value.clone())) => value
.as_str()
.and_then(ReasoningEffort::parse)
.and_then(|text| text.parse().ok())
.ok_or_else(|| unmapped_effort(value)),
None | Some(Recognized::Unrecognized(_)) => Ok(ReasoningEffort::Medium),
}

2
litellm-rust/crates/llms/src/anthropic/messages/transformation.rs
View file

@ -270,7 +270,7 @@ fn drop_unsupported_params(
fn speed_text(speed: &Recognized<Speed>) -> String {
match speed {
Recognized::Known(speed) => speed.as_str().to_string(),
Recognized::Known(speed) => <&'static str>::from(*speed).to_string(),
Recognized::Unrecognized(Value::String(text)) => text.clone(),
Recognized::Unrecognized(other) => other.to_string(),
}

1
litellm-rust/crates/llms/src/anthropic/mod.rs
View file

@ -1,3 +1,4 @@
pub mod beta_headers;
pub mod common_utils;
pub mod batches;

4
litellm-rust/crates/llms/src/aws_textract/ocr/common_utils.rs
View file

@ -23,9 +23,11 @@ const HEALTH_CHECK_IMAGE_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAA
/// Textract has operations rather than models; the model slot of
/// `aws_textract/<model>` names the one to call.
#[derive(Clone, Copy, Debug, EnumString, IntoStaticStr, VariantNames, PartialEq, Eq)]
#[strum(serialize_all = "kebab-case", ascii_case_insensitive)]
#[strum(ascii_case_insensitive)]
pub enum TextractOperation {
#[strum(serialize = "detect-document-text")]
DetectDocumentText,
#[strum(serialize = "analyze-document")]
AnalyzeDocument,
}

13
litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs
View file

@ -30,20 +30,17 @@ pub struct BedrockAudioTranscriptionConfig;
#[derive(Clone, Copy, Deserialize, IntoStaticStr)]
#[serde(rename_all = "lowercase")]
#[strum(serialize_all = "lowercase")]
enum AudioFormat {
#[strum(serialize = "wav")]
Wav,
#[strum(serialize = "mp3")]
Mp3,
#[strum(serialize = "flac")]
Flac,
#[strum(serialize = "ogg")]
Ogg,
}
impl AudioFormat {
fn as_str(self) -> &'static str {
self.into()
}
}
struct AudioInput {
data: String,
format: AudioFormat,
@ -118,7 +115,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
"messages": [{
"role": "user",
"content": [
{"audio": {"format": audio.format.as_str(), "source": {"bytes": audio.data}}},
{"audio": {"format": <&'static str>::from(audio.format), "source": {"bytes": audio.data}}},
{"text": instruction}
]
}],

2
litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs
View file

@ -385,7 +385,7 @@ fn converse_body(conversation: &Conversation, optional_params: &Map<String, Valu
.iter()
.map(|turn| {
json!({
"role": turn.role.as_str(),
"role": <&'static str>::from(turn.role),
"content": turn.texts.iter().map(|text| json!({"text": text})).collect::<Vec<_>>(),
})
})

38
litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs
View file

@ -64,27 +64,23 @@ pub fn invoke_anthropic_event_stream(bytes: ByteStream) -> EventStream {
Box::pin(invoke_chunk_stream(bytes).map(|chunk| decode_invoke_anthropic_chunk(chunk?)))
}
#[derive(Clone, Copy)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::EnumString)]
enum InvokeProvider {
#[strum(serialize = "anthropic")]
Anthropic,
#[strum(serialize = "deepseek_r1")]
DeepseekR1,
#[strum(serialize = "moonshot")]
Moonshot,
#[strum(disabled)]
Unsupported,
}
impl From<&str> for InvokeProvider {
fn from(value: &str) -> Self {
match value {
"anthropic" => Self::Anthropic,
"deepseek_r1" => Self::DeepseekR1,
"moonshot" => Self::Moonshot,
_ => Self::Unsupported,
}
}
}
pub fn invoke_chat_stream(invoke_provider: &str, shape: StreamShape) -> Result<ChatStream, Error> {
match InvokeProvider::from(invoke_provider) {
match invoke_provider
.parse()
.unwrap_or(InvokeProvider::Unsupported)
{
InvokeProvider::Anthropic => Ok(ChatStream::new(
invoke_anthropic_event_stream,
ModelResponseIterator::new(shape),
@ -116,6 +112,22 @@ mod tests {
const TEXT_DELTA: &str =
r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}"#;
#[rstest::rstest]
#[case::anthropic("anthropic", InvokeProvider::Anthropic)]
#[case::deepseek_r1("deepseek_r1", InvokeProvider::DeepseekR1)]
#[case::moonshot("moonshot", InvokeProvider::Moonshot)]
#[case::unknown("qwen", InvokeProvider::Unsupported)]
#[case::case_sensitive("Anthropic", InvokeProvider::Unsupported)]
fn invoke_provider_maps_each_model_family_name(
#[case] name: &str,
#[case] expected: InvokeProvider,
) {
assert_eq!(
name.parse().unwrap_or(InvokeProvider::Unsupported),
expected
);
}
fn aws_wire(chunk: &str) -> Vec<u8> {
let payload = serde_json::json!({"bytes": STANDARD.encode(chunk)});
let message = Message::new(Bytes::from(serde_json::to_vec(&payload).unwrap())).add_header(

10
litellm-rust/crates/llms/src/cohere/ocr/transformation.rs
View file

@ -25,11 +25,13 @@ const COHERE_API_KEY_ENV: &str = "COHERE_API_KEY";
const COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC";
#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize)]
#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, strum::IntoStaticStr)]
#[serde(rename_all = "lowercase")]
pub enum OutputFormat {
#[default]
#[strum(serialize = "markdown")]
Markdown,
#[strum(serialize = "blocks")]
Blocks,
}
@ -267,11 +269,7 @@ fn build_request(model: &str, image_url: String, params: &CohereOptions) -> Cohe
CohereRequest {
model: model.into(),
document: CohereParseDocument::ImageUrl { image_url },
output_format: match params.output_format.unwrap_or_default() {
OutputFormat::Markdown => "markdown",
OutputFormat::Blocks => "blocks",
}
.into(),
output_format: <&'static str>::from(params.output_format.unwrap_or_default()).into(),
}
}

111
litellm-rust/crates/llms/tests/anthropic_beta_headers.rs Normal file
View file

@ -0,0 +1,111 @@
use litellm_llms::anthropic::beta_headers::BetaPolicy;
use litellm_llms_types::providers::anthropic::BetaProvider;
use rstest::rstest;
fn headers(pairs: &[(&str, &str)]) -> Vec<(String, String)> {
pairs
.iter()
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect()
}
#[rstest]
fn forward_keeps_headers_as_is() {
let input = headers(&[
(
"Anthropic-Beta",
"example-beta-2099-01-01,web-search-2025-03-05",
),
("x-api-key", "k"),
]);
assert_eq!(BetaPolicy::Forward.apply(input.clone()), input);
}
#[rstest]
fn drop_removes_every_casing() {
assert_eq!(
BetaPolicy::Drop.apply(headers(&[
("anthropic-beta", "a"),
("x-api-key", "k"),
("ANTHROPIC-BETA", "b"),
])),
headers(&[("x-api-key", "k")])
);
}
#[rstest]
#[case::bedrock(BetaProvider::Bedrock, Some("tool-search-tool-2025-10-19"))]
#[case::bedrock_mantle(BetaProvider::BedrockMantle, Some("tool-search-tool-2025-10-19"))]
#[case::vertex_ai(BetaProvider::VertexAi, Some("tool-search-tool-2025-10-19"))]
#[case::anthropic(BetaProvider::Anthropic, Some("advanced-tool-use-2025-11-20"))]
#[case::azure_ai(BetaProvider::AzureAi, Some("advanced-tool-use-2025-11-20"))]
#[case::databricks(BetaProvider::Databricks, Some("advanced-tool-use-2025-11-20"))]
#[case::bedrock_converse(BetaProvider::BedrockConverse, None)]
fn filter_renames_per_host(#[case] provider: BetaProvider, #[case] expected_beta: Option<&str>) {
let expected = match expected_beta {
Some(beta) => headers(&[("x-api-key", "k"), ("anthropic-beta", beta)]),
None => headers(&[("x-api-key", "k")]),
};
assert_eq!(
BetaPolicy::Filter(provider).apply(headers(&[
("Anthropic-Beta", "advanced-tool-use-2025-11-20"),
("x-api-key", "k"),
])),
expected
);
}
#[rstest]
#[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")]
#[case::anthropic(BetaProvider::Anthropic, "bash_20241022")]
#[case::vertex_ai(BetaProvider::VertexAi, " , ")]
fn filter_removes_header_when_nothing_survives(
#[case] provider: BetaProvider,
#[case] beta_value: &str,
) {
assert_eq!(
BetaPolicy::Filter(provider).apply(headers(&[
("x-api-key", "k"),
("anthropic-beta", beta_value)
])),
headers(&[("x-api-key", "k")])
);
}
#[rstest]
#[case::anthropic(
BetaProvider::Anthropic,
"web-search-2025-03-05",
"oauth-2025-04-20,web-search-2025-03-05,example-beta-2099-01-01",
"oauth-2025-04-20,web-search-2025-03-05"
)]
#[case::vertex_ai_renames_and_deduplicates(
BetaProvider::VertexAi,
"advanced-tool-use-2025-11-20",
"tool-search-tool-2025-10-19",
"tool-search-tool-2025-10-19"
)]
fn filter_deduplicates_and_sorts(
#[case] provider: BetaProvider,
#[case] first_beta: &str,
#[case] second_beta: &str,
#[case] expected_beta: &str,
) {
assert_eq!(
BetaPolicy::Filter(provider).apply(headers(&[
("ANTHROPIC-BETA", first_beta),
("x-api-key", "k"),
("anthropic-beta", second_beta),
])),
headers(&[("x-api-key", "k"), ("anthropic-beta", expected_beta)])
);
}
#[rstest]
#[case::forward(BetaPolicy::Forward)]
#[case::drop(BetaPolicy::Drop)]
#[case::filter(BetaPolicy::Filter(BetaProvider::Bedrock))]
fn headers_without_a_beta_are_untouched(#[case] policy: BetaPolicy) {
let input = headers(&[("x-api-key", "k"), ("anthropic-version", "2023")]);
assert_eq!(policy.apply(input.clone()), input);
}

13
litellm-rust/crates/python-bridge/src/lib.rs
View file

@ -17,6 +17,10 @@ mod tokenizer;
#[pymodule(gil_used = true)]
mod _native {
#[pymodule_export]
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
use pyo3::{prelude::*, types::PyModule};
#[cfg(feature = "panic-test")]
#[pymodule_export]
use crate::diagnostics::_panic_for_test;
@ -29,9 +33,7 @@ mod _native {
#[pymodule_export]
use crate::routes::audio_transcription::{atranscription, transcription};
#[pymodule_export]
use crate::routes::chat_completions::{
achat_completions, acompletion, chat_completions, completion,
};
use crate::routes::chat_completions::{acompletion, completion};
#[pymodule_export]
use crate::routes::embeddings::{aembedding, embedding};
#[pymodule_export]
@ -51,9 +53,6 @@ mod _native {
use crate::tokenizer::HuggingFaceEncoding;
#[pymodule_export]
use crate::tokenizer::Tokenizer;
#[pymodule_export]
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
use pyo3::{prelude::*, types::PyModule};
#[pymodule_init]
fn init(module: &Bound<'_, PyModule>) -> PyResult<()> {
@ -101,8 +100,6 @@ mod tests {
"atranscription",
"messages",
"amessages",
"chat_completions",
"achat_completions",
"completion",
"acompletion",
"responses",

19
litellm-rust/crates/python-bridge/src/marshal.rs
View file

@ -19,13 +19,6 @@ pub(crate) struct RouteOptions {
pub(crate) timeout: Option<Duration>,
}
pub(crate) fn messages_argument(value: &Bound<'_, PyAny>) -> PyResult<Vec<Value>> {
match from_py_argument(value)? {
Value::Array(values) => Ok(values),
_ => Err(PyValueError::new_err("messages must be a list")),
}
}
fn required_object(name: &'static str, value: Value) -> PyResult<Map<String, Value>> {
match value {
Value::Object(values) => Ok(values),
@ -449,18 +442,6 @@ module.__getattr__ = fail
fn argument_converters_keep_nested_values_and_accept_explicit_none() {
Python::initialize();
Python::attach(|py| {
let messages = py
.eval(
c"[{'role': 'user', 'content': [{'type': 'text', 'text': 'hi'}]}]",
None,
None,
)
.unwrap();
assert_eq!(
Value::Array(messages_argument(&messages).unwrap()),
json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}])
);
let params = py.eval(c"{'temperature': 0.2}", None, None).unwrap();
assert_eq!(
optional_object("optional_params", &params).unwrap(),

16
litellm-rust/crates/python-bridge/src/python_settings.rs
View file

@ -6,13 +6,16 @@ use crate::coercion::{FieldSpec, ProjectionError};
const MODULE: &str = "litellm.rust_bridge.settings";
#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq)]
#[strum(serialize_all = "snake_case")]
pub(crate) enum PythonSettings {
#[strum(serialize = "http_settings")]
Http,
#[strum(serialize = "url_policy")]
UrlPolicy,
#[strum(serialize = "provider_defaults")]
ProviderDefaults,
#[strum(serialize = "secret_manager")]
SecretManager,
#[strum(serialize = "secret_manager_binding")]
SecretManagerBinding,
}
@ -23,17 +26,16 @@ pub(crate) struct Snapshot<'py> {
impl Snapshot<'_> {
pub(crate) fn read<T>(&self, spec: &FieldSpec<T>) -> Result<T, ProjectionError> {
spec.read(&self.value, self.group.name())
spec.read(&self.value, self.group.into())
}
}
impl PythonSettings {
pub(crate) fn name(self) -> &'static str {
self.into()
}
pub(crate) fn read(self, py: Python<'_>) -> PyResult<Snapshot<'_>> {
let value = py.import(MODULE)?.getattr(self.name())?.call0()?;
let value = py
.import(MODULE)?
.getattr(<&'static str>::from(self))?
.call0()?;
Ok(Snapshot { group: self, value })
}

4
litellm-rust/crates/python-bridge/src/routes/AGENTS.md
View file

@ -11,3 +11,7 @@ The host driver owns sequencing and terminal events; the bridge supplies fallibl
Use the shared `run_public_call` boundary with hooks supplied by bridge composition. `callbacks-legacy-python` owns legacy argument sharing and `Logging` dispatch behind `PublicCall` and `LegacyLogging`. Route bindings supply `callbacks-legacy-python::LoggingOperation` when composing legacy logging and may retain the request needed for projection, but must not duplicate the legacy callback contract
Regression tests must observe that an unstarted call does no setup, hook and preflight rewrites affect resource configuration, setup failures reach the selected failure handler once, and provider work is not replayed. Retain existing read-point and object-identity guarantees while changing setup timing
## Layout
`messages/` is the reference shape. Each lifecycle route is a folder: `mod.rs` holds `run_<route>` and the thin sync and async `#[pyfunction]` entrypoints that take `NativeCall` and call it, `host.rs` holds the `PythonBinding` host, and anything else route-specific (projection, error mapping, extra pyclasses) gets its own sibling file. Scaffolded routes without a lifecycle (embeddings, audio transcription) stay a single `<route>.rs`

24
litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs
View file

@ -5,6 +5,8 @@ use litellm_inference_transcription::{
use pyo3::prelude::*;
use serde_json::{Map, Value};
use super::NativeCall;
use crate::{
errors::route_error_to_pyerr,
marshal::{RouteOptions, optional_object_field, required_field, value_route_options},
@ -40,13 +42,12 @@ async fn execute(
}
#[pyfunction]
pub(crate) fn transcription(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
pub(crate) fn transcription(py: Python<'_>, call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
let arguments = call.resolved()?;
let audio: Value =
litellm_host_python::from_py_argument(&required_field(&call.bound, "audio")?)?;
let options = value_route_options(&call.bound)?;
let optional_params =
optional_object_field(&call.bound, "optional_params")?.unwrap_or_default();
litellm_host_python::from_py_argument(&required_field(&arguments, "audio")?)?;
let options = value_route_options(&arguments)?;
let optional_params = optional_object_field(&arguments, "optional_params")?.unwrap_or_default();
let http = crate::http::provider_client(py, &call.kwargs, false)?;
let secrets = crate::secrets::source(py)?;
run_sync(
@ -59,14 +60,13 @@ pub(crate) fn transcription(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<
#[pyfunction]
pub(crate) fn atranscription<'py>(
py: Python<'py>,
call: Bound<'py, PyAny>,
call: NativeCall<'py>,
) -> PyResult<Bound<'py, PyAny>> {
let call = super::NativeCall::extract(&call)?;
let arguments = call.resolved()?;
let audio: Value =
litellm_host_python::from_py_argument(&required_field(&call.bound, "audio")?)?;
let options = value_route_options(&call.bound)?;
let optional_params =
optional_object_field(&call.bound, "optional_params")?.unwrap_or_default();
litellm_host_python::from_py_argument(&required_field(&arguments, "audio")?)?;
let options = value_route_options(&arguments)?;
let optional_params = optional_object_field(&arguments, "optional_params")?.unwrap_or_default();
let http = crate::http::provider_client(py, &call.kwargs, true)?;
let secrets = crate::secrets::source(py)?;
run_async(

150
litellm-rust/crates/python-bridge/src/routes/chat_completions.rs
View file

@ -1,150 +0,0 @@
mod host;
use pyo3::types::{PyDict, PyTuple};
use crate::execution::{run_async, run_sync};
use litellm_inference_chat::{ChatCompletionsRoute, Error, types::ChatCompletionsRequest};
use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
use pyo3::prelude::*;
use serde_json::{Map, Value};
use crate::{
errors::route_error_to_pyerr,
marshal::{
RouteOptions, messages_argument, optional_object_field, required_field, value_route_options,
},
};
async fn execute(
http: Result<litellm_http::Client, litellm_http::Error>,
secrets: std::sync::Arc<dyn litellm_secrets::source::SecretSource>,
messages: Vec<Value>,
optional_params: Map<String, Value>,
options: RouteOptions,
) -> Result<ChatCompletionsResponse, Error> {
let RouteOptions {
model,
api_key,
api_base,
custom_llm_provider,
extra_headers,
timeout,
} = options;
ChatCompletionsRoute::new(http?, crate::http::resources().auth.clone(), secrets)
.execute(
ChatCompletionsRequest {
model: &model,
messages: Value::Array(messages),
optional_params,
api_key: api_key.as_deref(),
api_base: api_base.as_deref(),
custom_llm_provider: custom_llm_provider.as_deref(),
extra_headers,
timeout,
},
&(),
None,
)
.await
}
#[pyfunction]
pub(crate) fn chat_completions(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
let messages: Vec<Value> = messages_argument(&required_field(&call.bound, "messages")?)?;
let optional_params =
optional_object_field(&call.bound, "optional_params")?.unwrap_or_default();
let options = value_route_options(&call.bound)?;
let http = crate::http::provider_client(py, &call.kwargs, false)?;
let secrets = crate::secrets::source(py)?;
run_sync(
py,
execute(http, secrets, messages, optional_params, options),
route_error_to_pyerr,
)
}
#[pyfunction]
pub(crate) fn achat_completions<'py>(
py: Python<'py>,
call: Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
let call = super::NativeCall::extract(&call)?;
let messages: Vec<Value> = messages_argument(&required_field(&call.bound, "messages")?)?;
let optional_params =
optional_object_field(&call.bound, "optional_params")?.unwrap_or_default();
let options = value_route_options(&call.bound)?;
let http = crate::http::provider_client(py, &call.kwargs, true)?;
let secrets = crate::secrets::source(py)?;
run_async(
py,
execute(http, secrets, messages, optional_params, options),
route_error_to_pyerr,
)
}
fn run_public(
py: Python<'_>,
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
asynchronous: bool,
) -> PyResult<Py<PyAny>> {
use super::inference::InferenceHost;
use litellm_callbacks_legacy_python::LoggingOperation;
let host = InferenceHost::new(
request.clone().unbind(),
"litellm.rust_bridge.chat_completions.route_host",
);
let cache_call_type = if asynchronous {
"acompletion"
} else {
"completion"
};
crate::cache::admit_native(py, &kwargs, cache_call_type)?;
let (arguments, hooks) = crate::routes::call_hooks(
py,
LoggingOperation::Completion,
&request,
&args,
&kwargs,
asynchronous,
)?;
crate::routes::run_public_call(
py,
arguments,
move |py, arguments, request| {
let route = ChatCompletionsRoute::new(
crate::http::provider_client(py, arguments, asynchronous)?
.map_err(crate::http::client_error)?,
crate::http::resources().auth.clone(),
crate::secrets::source(py)?,
);
let (cache, cache_options) =
crate::cache::configured_native(py, arguments, cache_call_type)?;
let route = match cache {
Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new(
cache,
litellm_cache_response::CacheScope::Shared,
)),
None => route,
};
Ok(route.machine(request, cache_options.policy))
},
host::ChatCompletionsPythonHost(host),
hooks,
asynchronous,
)
}
#[pyfunction]
pub(crate) fn completion(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_public(py, call.bound.into_any(), call.args, call.kwargs, false)
}
#[pyfunction]
pub(crate) fn acompletion(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let call = super::NativeCall::extract(&call)?;
run_public(py, call.bound.into_any(), call.args, call.kwargs, true)
}

2
litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs
View file

@ -1,6 +1,6 @@
use std::convert::Infallible;
use super::super::inference::InferenceHost;
use crate::routes::inference::InferenceHost;
use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned};
use litellm_inference_chat::{Error, route::ChatCompletions, types::ChatCompletionsCall};
use pyo3::{

71
litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs Normal file
View file

@ -0,0 +1,71 @@
mod host;
use std::sync::Arc;
use host::ChatCompletionsPythonHost;
use litellm_auth::AuthServices;
use litellm_cache_response::{CachePolicy, ScopedCache};
use litellm_callbacks_legacy_python::LoggingOperation;
use litellm_host::{call::HostedMachine, protocol::Protocol};
use litellm_inference_chat::{ChatCompletionsRoute, route::ChatCompletions};
use litellm_secrets::source::SecretSource;
use pyo3::prelude::*;
use super::{
NativeCall,
inference::{InferenceHost, InferenceRoute, run_inference},
};
fn run_chat_completions(
py: Python<'_>,
call: NativeCall<'_>,
asynchronous: bool,
) -> PyResult<Py<PyAny>> {
let host = InferenceHost::new(
call.resolved()?.unbind(),
"litellm.rust_bridge.chat_completions.route_host",
);
run_inference::<ChatCompletionsRoute, _>(
py,
call,
asynchronous,
ChatCompletionsPythonHost(host),
)
}
#[pyfunction]
pub(crate) fn completion(py: Python<'_>, call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
run_chat_completions(py, call, false)
}
#[pyfunction]
pub(crate) fn acompletion(py: Python<'_>, call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
run_chat_completions(py, call, true)
}
impl InferenceRoute for ChatCompletionsRoute {
type Protocol = ChatCompletions;
const OPERATION: LoggingOperation = LoggingOperation::Completion;
const SYNC_CALL_TYPE: &'static str = "completion";
const ASYNC_CALL_TYPE: &'static str = "acompletion";
fn new(
http: litellm_http::Client,
auth: Arc<AuthServices>,
secrets: Arc<dyn SecretSource>,
) -> Self {
Self::new(http, auth, secrets)
}
fn with_cache(self, cache: ScopedCache) -> Self {
self.with_cache(cache)
}
fn machine(
self,
call: <ChatCompletions as Protocol>::Request,
policy: CachePolicy,
) -> HostedMachine<ChatCompletions> {
self.machine(call, policy)
}
}

12
litellm-rust/crates/python-bridge/src/routes/embeddings.rs
View file

@ -1,17 +1,18 @@
use pyo3::prelude::*;
use super::NativeCall;
use crate::errors::RustBridgeDeclined;
#[pyfunction]
pub(crate) fn embedding(call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
drop(super::NativeCall::extract(&call)?);
pub(crate) fn embedding(call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
drop(call);
Err(RustBridgeDeclined::new_err(
"native embeddings route is not implemented",
))
}
#[pyfunction]
pub(crate) fn aembedding(call: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
pub(crate) fn aembedding(call: NativeCall<'_>) -> PyResult<Py<PyAny>> {
embedding(call)
}
@ -31,12 +32,13 @@ mod tests {
let locals = PyDict::new(py);
py.run(
c"from types import SimpleNamespace
call = SimpleNamespace(args=(), kwargs={}, bound={'model':'test-model','input':'hello'})",
call = SimpleNamespace(args=(), kwargs={'model':'test-model','input':'hello'}, base={})",
Some(&locals),
Some(&locals),
)
.unwrap();
let call = locals.get_item("call").unwrap().unwrap();
let call: super::NativeCall<'_> =
locals.get_item("call").unwrap().unwrap().extract().unwrap();
let error = if asynchronous {
super::aembedding(call)
} else {

89
litellm-rust/crates/python-bridge/src/routes/inference.rs
View file

@ -1,11 +1,19 @@
use std::sync::Arc;
use litellm_auth::AuthServices;
use litellm_cache_response::{CachePolicy, CacheScope, ScopedCache};
use litellm_callbacks_legacy_python::LoggingOperation;
use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
use litellm_host_python::{from_py, lookup};
use litellm_host::{call::HostedMachine, protocol::Protocol};
use litellm_host_python::{PythonBinding, PythonHostCalls, from_py, present};
use litellm_http::transport::Error as TransportError;
use litellm_inference::RouteError;
use litellm_secrets::source::SecretSource;
use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
use serde::Serialize;
use serde_json::{Map, Value};
use super::NativeCall;
use crate::{
errors::{RustUpstreamError, route_error_to_pyerr},
marshal::{
@ -15,7 +23,7 @@ use crate::{
};
pub(super) struct InferenceHost {
pub request: Py<PyAny>,
pub request: Py<PyDict>,
module: &'static str,
}
@ -26,7 +34,7 @@ pub(super) struct ProjectedCall {
}
impl InferenceHost {
pub fn new(request: Py<PyAny>, module: &'static str) -> Self {
pub fn new(request: Py<PyDict>, module: &'static str) -> Self {
Self { request, module }
}
@ -85,21 +93,7 @@ impl InferenceHost {
arguments: &Bound<'py, PyDict>,
name: &str,
) -> PyResult<Option<Bound<'py, PyAny>>> {
let request = self.request.bind(py);
if let Some(value) = lookup(arguments, request, name)? {
return Ok((!value.is_none()).then_some(value));
}
if request.is_instance_of::<PyDict>() {
return Ok(None);
}
let parameter = request
.getattr("parameters")?
.call_method1("get", (name,))?;
if !parameter.is_none() {
return Ok(Some(parameter));
}
let extra = request.getattr("kwargs")?.call_method1("get", (name,))?;
Ok((!extra.is_none()).then_some(extra))
present(arguments, self.request.bind(py), name)
}
pub fn parameters(
@ -140,3 +134,62 @@ impl InferenceHost {
Ok(PyErr::from_value(mapped))
}
}
pub(super) trait InferenceRoute: Sized + 'static {
type Protocol: Protocol<Error = RouteError>;
const OPERATION: LoggingOperation;
const SYNC_CALL_TYPE: &'static str;
const ASYNC_CALL_TYPE: &'static str;
fn new(
http: litellm_http::Client,
auth: Arc<AuthServices>,
secrets: Arc<dyn SecretSource>,
) -> Self;
fn with_cache(self, cache: ScopedCache) -> Self;
fn machine(
self,
call: <Self::Protocol as Protocol>::Request,
policy: CachePolicy,
) -> HostedMachine<Self::Protocol>;
}
pub(super) fn run_inference<R, H>(
py: Python<'_>,
call: NativeCall<'_>,
asynchronous: bool,
host: H,
) -> PyResult<Py<PyAny>>
where
R: InferenceRoute,
H: PythonBinding<Protocol = R::Protocol> + PythonHostCalls<R::Protocol> + 'static,
{
let call_type = if asynchronous {
R::ASYNC_CALL_TYPE
} else {
R::SYNC_CALL_TYPE
};
crate::cache::admit_native(py, &call.kwargs, call_type)?;
let (arguments, hooks) = super::call_hooks(py, R::OPERATION, &call, asynchronous)?;
super::run_public_call(
py,
arguments,
move |py, arguments, request| {
let route = R::new(
crate::http::provider_client(py, arguments, asynchronous)?
.map_err(crate::http::client_error)?,
crate::http::resources().auth.clone(),
crate::secrets::source(py)?,
);
let (cache, cache_options) = crate::cache::configured_native(py, arguments, call_type)?;
let route = match cache {
Some(cache) => route.with_cache(ScopedCache::new(cache, CacheScope::Shared)),
None => route,
};
Ok(route.machine(request, cache_options.policy))
},
host,
hooks,
asynchronous,
)
}

Some files were not shown because too many files have changed in this diff Show more