scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-18 04:44:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 125

Merge remote-tracking branch 'origin/main' into litellm_spend_counter_ttl

Browse source
This commit is contained in:
mateo-berri 2026-10-13 10:57:29 -07:00
commit 9535a2b34b
638 changed files with 49755 additions and 6940 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

216
.circleci/config.yml
View file

@ -38,7 +38,7 @@ commands:
parameters:
category:
type: enum
enum: ["backend", "client", "redis-compat"]
enum: ["backend", "client"]
default: "backend"
steps:
- run:
@ -339,8 +339,15 @@ commands:
Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`""
}
setup_litellm_test_deps:
parameters:
checkout:
type: boolean
default: true
steps:
- checkout
- when:
condition: << parameters.checkout >>
steps:
- checkout
- setup_google_dns
- install_uv
- install_rust
@ -376,13 +383,28 @@ jobs:
shell: bash.exe
command: bash .circleci/scripts/path_filter.sh windows-release
- run:
name: Install Python
name: Install Python 3.11.9 from python.org
no_output_timeout: 6m
command: |
choco install python --version=3.11.0 -y --no-progress --force
refreshenv
python --version
environment:
CHOCOLATEY_CONFIRM_ALL: "true"
$ProgressPreference = "SilentlyContinue"
$installer = Join-Path $env:TEMP "python-3.11.9-amd64.exe"
Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe -OutFile $installer -TimeoutSec 120
$expected = "5ee42c4eee1e6b4464bb23722f90b45303f79442df63083f05322f1785f5fdde"
$actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower()
if ($actual -ne $expected) {
throw "Python installer hash mismatch: expected $expected got $actual"
}
$arguments = "/quiet InstallAllUsers=1 PrependPath=1 Include_test=0 Include_doc=0 Include_launcher=0 TargetDir=C:\Python311"
$process = Start-Process -FilePath $installer -ArgumentList $arguments -PassThru
if (-not $process.WaitForExit(300000)) {
$process.Kill()
throw "Python installer did not finish within 5 minutes"
}
if ($process.ExitCode -ne 0) {
throw "Python installer exited with $($process.ExitCode)"
}
Remove-Item $installer
& C:\Python311\python.exe --version
- install_windows_toolchain
- run:
name: Record the Rust build environment for the release cargo cache key
@ -405,7 +427,7 @@ jobs:
environment:
UV_HTTP_TIMEOUT: "300"
command: |
$env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
$env:Path = "C:\Python311;C:\Python311\Scripts;$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
uv build --wheel --out-dir dist
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
@ -512,11 +534,12 @@ jobs:
--set image.repository=litellm-ci \
--set image.tag=${IMAGE_TAG} \
--set image.pullPolicy=Never
# Wait for pod to be ready
echo "Waiting 30 seconds for pod to be ready..."
sleep 30
if ! kubectl rollout status deployment/litellm --timeout=180s; then
kubectl describe pods -l app.kubernetes.io/name=litellm
kubectl logs -l app.kubernetes.io/name=litellm --tail=200
exit 1
fi
# Print pod logs before running tests
echo "Printing pod logs..."
kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
@ -1260,9 +1283,19 @@ jobs:
- run:
name: Build the candidate dashboard
command: cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build
- when:
condition:
equal: [mcp, << parameters.suite >>]
steps:
- run:
name: Install MCP SDK1 peer
command: |
uv venv --python 3.12 .venv-mcp-peer
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
- start_postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
server_args: "-c max_connections=300 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
- start_redis
- run:
name: Run owned integration contracts
@ -1315,12 +1348,27 @@ jobs:
default: standard
shards:
type: integer
machine:
image: ubuntu-2204:2024.04.1
resource_class: medium
docker:
- image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
resource_class: small
working_directory: ~/project
environment:
UV_PYTHON_DOWNLOADS: never
steps:
- setup_litellm_test_deps
- checkout
- install_uv
- run:
name: Install the locked shard verifier and coverage dependencies
command: |
uv export --frozen --all-groups --all-extras --no-emit-project --no-emit-workspace --no-header --no-annotate \
| awk '/^[^ ]/ { keep = $0 ~ /^(annotated-types|coverage|pydantic|pydantic-core|typing-extensions|typing-inspection)==/ } keep' \
> /tmp/fan-in-requirements.txt
test "$(grep -c '^[^ ]' /tmp/fan-in-requirements.txt)" -eq 6
uv venv .venv --python 3.12
uv pip install --no-config --python .venv --require-hashes -r /tmp/fan-in-requirements.txt
- attach_workspace:
at: /tmp/integration-shards
- run:
@ -1436,7 +1484,8 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_litellm_test_deps
- setup_litellm_test_deps:
checkout: false
- start_postgres:
db_name: litellm_test
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
@ -1481,108 +1530,6 @@ jobs:
- store_artifacts:
path: test-results
mcp_integration:
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
environment:
COVERAGE_CORE: sysmon
LITELLM_LOCAL_MODEL_COST_MAP: "True"
steps:
- checkout
- skip_if_unrelated_changes
- setup_litellm_test_deps
- run:
name: Generate Prisma client
command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- run:
name: Install MCP SDK1 peer
command: |
uv venv --python 3.12 .venv-mcp-peer
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
- run:
name: Run MCP integration tests
command: |
mkdir -p test-results
env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY \
timeout --signal=TERM 20m uv run --no-sync pytest \
tests/mcp_tests tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \
tests/unit/responses/mcp --tb=short -vv --maxfail=10 -n 2 --dist=loadscope --reruns 0 \
--reruns-delay 1 --timeout=120 --rerun-except "from pytest-timeout" --durations=20 \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml \
--cov-config=pyproject.toml -o junit_family=xunit1 --junitxml=test-results/junit.xml
- install_codecov_cli
- run:
name: Upload MCP integration coverage
when: always
command: |
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
-F mcp-integration -C "$CIRCLE_SHA1" \
-n "mcp-integration-${CIRCLE_BUILD_NUM}" --git-service github
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
redis_compat:
parameters:
redis_py:
type: string
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes:
category: redis-compat
- setup_litellm_test_deps
- run:
name: Pin redis-py version
command: |
uv pip install "redis==<< parameters.redis_py >>"
uv run --no-sync python -c "import redis; assert redis.__version__ == '<< parameters.redis_py >>', redis.__version__; print('redis-py', redis.__version__)"
- run:
name: Install redis-server 7.2.16
command: |
mkdir -p "$HOME/.local/bin"
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
docker cp "${cid}":/usr/local/bin/redis-server "$HOME/.local/bin/redis-server"
docker rm "$cid"
redis-server --version | grep -q 'v=7.2.16'
- run:
name: Run Redis compatibility tests
command: |
mkdir -p test-results
env -u CASSETTE_REDIS_URL -u AZURE_CLIENT_ID -u AZURE_CLIENT_SECRET -u AZURE_TENANT_ID \
timeout --signal=TERM 15m uv run --no-sync pytest \
tests/unit/test_redis.py tests/unit/caching/test_redis_connection_pool.py \
tests/unit/caching/test_redis_cluster_cache.py tests/unit/caching/test_evicted_client_closer.py \
tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_azure_credentials \
tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_gcp_credentials \
--tb=short -vv --reruns 2 --reruns-delay 1 --durations=20 --cov=./litellm \
--cov-report=xml:coverage.xml -o junit_family=xunit1 --junitxml=test-results/junit.xml
- when:
condition:
equal: ["5.3.1", << parameters.redis_py >>]
steps:
- install_codecov_cli
- run:
name: Upload Redis compatibility coverage
when: always
command: |
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
-F redis-compat -C "$CIRCLE_SHA1" \
-n "redis-compat-${CIRCLE_BUILD_NUM}" --git-service github
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
workflows:
migration_startup_scheduled:
triggers:
@ -1682,7 +1629,6 @@ workflows:
- integration_contracts:
name: integration-mcp
suite: mcp
resource_class: xlarge
parallelism: 2
filters: *integration_default_run
- integration_shard_coverage:
@ -1732,19 +1678,6 @@ workflows:
shards: 2
requires: [integration-database-replica]
filters: *integration_default_run
- postgres_suite:
name: proxy-security
test_path: tests/proxy_security_tests
seed: true
timeout_minutes: 15
filters: *integration_default_run
- postgres_suite:
name: schema-migration
test_path: tests/proxy_migration_tests
seed: false
timeout_minutes: 20
workers: 2
filters: *integration_default_run
- postgres_suite:
name: roi-database
test_path: tests/integration/database/test_roi_observed.py
@ -1752,19 +1685,6 @@ workflows:
coverage_flag: roi-postgres
timeout_minutes: 10
filters: *integration_default_run
- mcp_integration:
name: mcp-integration
filters: *integration_default_run
- redis_compat:
name: redis-compat-<< matrix.redis_py >>
matrix:
parameters:
redis_py:
- "5.3.1"
- "6.4.0"
- "7.4.1"
- "8.0.1"
filters: *integration_default_run
- windows_release_wheel:
filters: *integration_default_run
- base_sdk_install:

2
.circleci/scripts/classify_changes.sh
View file

@ -14,7 +14,7 @@ outside_cost_map_set=false
while IFS= read -r file || [ -n "$file" ]; do
[ -n "$file" ] || continue
case "$file" in
litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock)
litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .circleci/scripts/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock)
has_redis_compat=true ;;
esac
case "$file" in

2
.circleci/scripts/path_filter.sh
View file

@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -uo pipefail
category="${1:?usage: path_filter.sh <backend|client|redis-compat>}"
category="${1:?usage: path_filter.sh <backend|client|windows-release>}"
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
run_full() {

44
.circleci/scripts/run_integration.sh
View file

@ -128,20 +128,14 @@ fi
sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
isolated_workers=false
if [ "$mode" = parity ]; then
export INTEGRATION_WORKERS=1
else
case "$suite" in
management|accounting) export INTEGRATION_WORKERS=2; isolated_workers=true ;;
providers|extensions) export INTEGRATION_WORKERS=4; isolated_workers=true ;;
management|accounting) export INTEGRATION_WORKERS=2 ;;
providers|extensions) export INTEGRATION_WORKERS=4 ;;
esac
fi
if [ "$isolated_workers" = false ]; then
setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
.venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 &
upstream_pid=$!
fi
if [ "$suite" = cost ]; then
export INTEGRATION_WORKERS=8
fi
@ -149,7 +143,16 @@ if [ "$suite" = security ]; then
export INTEGRATION_WORKERS=4
fi
if [ "$suite" = mcp ]; then
export INTEGRATION_WORKERS=4 INTEGRATION_COVERAGE=1
export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
fi
isolated_workers=false
if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
isolated_workers=true
fi
if [ "$isolated_workers" = false ]; then
setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
.venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 &
upstream_pid=$!
fi
coverage_data="$PWD/$results/coverage/data"
proxy_command=(.venv/bin/python -m integration._support.proxy)
@ -191,9 +194,6 @@ start_proxy() {
> "$results/$log_name" 2>&1 &
launched_pid=$!
}
if { [ "$suite" = management ] || [ "$suite" = mcp ]; } && [ "$isolated_workers" = true ]; then
export INTEGRATION_PEER_URL=http://127.0.0.1:4001
fi
if [ "$isolated_workers" = false ]; then
start_proxy 4000 proxy.log
proxy_pid="$launched_pid"
@ -239,13 +239,24 @@ if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then
shard_arguments=(--shard-files "$results/node-files.txt")
fi
shared_services=()
worker_coverage=()
if [ "$isolated_workers" = false ]; then
shared_services=(
"REDIS_HOST=$REDIS_HOST" "REDIS_PORT=$REDIS_PORT"
"INTEGRATION_PROXY_URL=$INTEGRATION_PROXY_URL" "INTEGRATION_PEER_URL=$INTEGRATION_PEER_URL"
"INTEGRATION_UPSTREAM_URL=$INTEGRATION_UPSTREAM_URL" "INTEGRATION_MASTER_KEY=$INTEGRATION_MASTER_KEY"
)
elif [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
worker_coverage=("INTEGRATION_COVERAGE_FILE=$coverage_data")
fi
env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
INTEGRATION_RUN_ID="$integration_identity" \
DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
INTEGRATION_PROXY_URL="$INTEGRATION_PROXY_URL" INTEGRATION_PEER_URL="$INTEGRATION_PEER_URL" \
INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
DATABASE_URL="$DATABASE_URL" \
"${shared_services[@]}" "${worker_coverage[@]}" \
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
INTEGRATION_MASTER_KEY="$INTEGRATION_MASTER_KEY" LITELLM_MODE=PRODUCTION \
LITELLM_MODE=PRODUCTION \
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
INTEGRATION_SEED="$INTEGRATION_SEED" \
INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
@ -254,6 +265,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \
INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \
INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
MCP_TEST_PEER_PYTHON="${MCP_TEST_PEER_PYTHON:-}" \
.venv/bin/python tests/integration/run.py "$suite" --results "$results" \
"${shard_arguments[@]}"

2
.github/PULL_REQUEST_TEMPLATE/general.md vendored
View file

@ -68,7 +68,7 @@ After: the same request comes back with real token counts, so the dashboard show
- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/unit/<your_test_file>.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more
- [ ] My PR passes all required CI/CD checks (e.g., lint, schema.d.ts sync check, etc.)
- [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
- [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes)
- [ ] Every automated review on my PR's latest commit has finished, and I have addressed or replied to each of its findings before requesting a maintainer review
## Delays in PR merge?

14
.github/workflows/image-scan.yml vendored
View file

@ -27,7 +27,7 @@ on:
- backend/routes/allowlist.py
- pyproject.toml
- litellm-proxy-extras/**
- tests/proxy_migration_tests/**
- tests/integration/database/image/**
- uv.lock
- ui/litellm-dashboard/package-lock.json
- ui/Dockerfile
@ -92,7 +92,7 @@ jobs:
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
# Scans the whole shipped artifact: OS/apk plus every language package
# baked into the image, including ones no lockfile declares (e.g. prisma's
@ -145,7 +145,7 @@ jobs:
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
migrations-image:
@ -177,7 +177,7 @@ jobs:
LITELLM_MIGRATION_SCRIPT: /app/run.py
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py -v
gateway-image:
name: gateway-image
@ -208,7 +208,7 @@ jobs:
LITELLM_IMAGE_COMPONENT: gateway
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
ui-image:
name: ui-image
@ -237,7 +237,7 @@ jobs:
LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
python -m pytest --noconftest tests/integration/database/image/test_ui_image_serves_offline.py -v
backend-image:
name: backend-image
@ -272,4 +272,4 @@ jobs:
LITELLM_IMAGE_COMPONENT: backend
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_admin_mcp.py -v

4
.github/workflows/test-e2e-redis-chaos.yml vendored
View file

@ -91,12 +91,12 @@ jobs:
- name: Run the Redis chaos load test
env:
E2E_REDIS_CHAOS: "1"
E2E_PERF_TESTS: "1"
LITELLM_PROXY_URL: http://localhost:4000
REDIS_HOST: 127.0.0.1
REDIS_PORT: "6379"
run: |
uv run --no-sync pytest tests/e2e/load/test_redis_chaos_e2e.py -v --tb=short -rA -s
uv run --no-sync pytest tests/e2e/perf/test_redis_chaos_e2e.py -v --tb=short -rA -s
- name: Show proxy log on failure
if: failure()

6
.github/workflows/test-linting.yml vendored
View file

@ -615,12 +615,6 @@ jobs:
- name: check_migrations_no_data_rewrites
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
- name: check_greptile_files_json
run: uv run --no-sync python ./tests/code_coverage_tests/check_greptile_files_json.py
- name: test_check_greptile_files_json
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_greptile_files_json.py
- name: check_no_publicly_known_master_key
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py

147
.github/workflows/test-unit.yml vendored
View file

@ -794,6 +794,10 @@ jobs:
tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
tests/unit/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py
tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py
tests/unit/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py
tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py
tests/unit/proxy/_experimental/mcp_server/outbound_credentials
python-version: "3.12"
workers: 4
@ -802,6 +806,28 @@ jobs:
job-timeout-minutes: 60
dist: loadscope
- shard: mcp-server
test-path: |-
tests/unit/experimental_mcp_client
tests/unit/proxy/_experimental/mcp_server
tests/unit/responses/mcp
--ignore=tests/unit/experimental_mcp_client/test_mcp_client.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_capabilities.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_interactions.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_operations.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
--ignore=tests/unit/proxy/_experimental/mcp_server/outbound_credentials
python-version: "3.12"
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
dist: loadscope
- shard: custom-logging
test-path: |-
tests/unit/proxy/test_proxy_server_custom_logger.py
@ -1324,6 +1350,115 @@ jobs:
- name: Check consumer types with the installed SDK
if: steps.changes.outputs.decision != 'skip'
run: uv run --no-sync basedpyright --pythonpath .venv/bin/python --project tests/typing/pyrightconfig.json
redis-compat:
name: redis-py ${{ matrix.redis-py }} compatibility
needs: rust-bridge
if: ${{ !cancelled() }}
permissions:
contents: read
id-token: write
pull-requests: read
runs-on: ubuntu-latest
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
redis-py: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"]
env:
UV_PYTHON: "3.12"
LITELLM_LOCAL_MODEL_COST_MAP: "True"
REDIS_PY_VERSION: ${{ matrix.redis-py }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
timeout-minutes: 3
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
timeout-minutes: 2
uses: ./.github/actions/detect-changes
with:
category: redis-compat
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ env.UV_PYTHON }}
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/cache-uv-downloads
- name: Set up the Rust build
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/rust-bridge
with:
artifact: ${{ needs.rust-bridge.outputs.artifact }}
- name: Install dependencies with redis-py ${{ matrix.redis-py }}
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 8
env:
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
run: |
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
fi
uv pip install --python .venv/bin/python "redis==${REDIS_PY_VERSION:?}"
uv run --no-sync python -c 'import os, redis; assert redis.__version__ == os.environ["REDIS_PY_VERSION"], redis.__version__'
- name: Install redis-server 7.2.16
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
run: |
mkdir -p "$RUNNER_TEMP/redis-bin"
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
docker rm "$cid"
"$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
- name: Run Redis compatibility tests
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 15
env:
COVERAGE_CORE: sysmon
run: |
uv run --no-sync pytest \
tests/unit/test_redis.py \
tests/unit/caching/test_redis_connection_pool.py \
tests/unit/caching/test_redis_cluster_cache.py \
tests/unit/caching/test_evicted_client_closer.py \
tests/integration/sdk/test_redis_cluster_iam_auth.py \
--tb=short -vv -rs -p no:rerunfailures -p no:pytest-retry --timeout=120 --durations=20 \
--cov=./litellm --cov-report=xml:coverage.xml
- name: Upload redis-py ${{ matrix.redis-py }} coverage to Codecov
if: always() && steps.changes.outputs.decision != 'skip' && matrix.redis-py == '5.3.1'
continue-on-error: true
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
files: coverage.xml
root_dir: ${{ github.workspace }}
flags: redis-compat
fail_ci_if_error: false
coverage:
name: coverage
needs: unit
@ -1369,10 +1504,20 @@ jobs:
root_dir: ${{ github.workspace }}
flags: unit
fail_ci_if_error: false
- name: Upload MCP server coverage under its own flag
continue-on-error: true
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
files: coverage-reports/coverage-mcp-server-${{ github.run_id }}-${{ github.run_attempt }}/coverage.xml
root_dir: ${{ github.workspace }}
flags: mcp-integration
fail_ci_if_error: false
unit-passed:
name: unit passed
permissions: {}
needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs, helm, openai-sdk-compat]
needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs, helm, openai-sdk-compat, redis-compat]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 2

4
.github/workflows/weekly_load_anomaly.yml vendored
View file

@ -63,7 +63,7 @@ jobs:
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/load/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
nohup uv run --no-sync litellm --config tests/e2e/perf/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
@ -78,7 +78,7 @@ jobs:
env:
E2E_WEEKLY_ANOMALY: "1"
run: |
uv run --no-sync pytest tests/e2e/load/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
uv run --no-sync pytest tests/e2e/perf/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
- name: Show proxy log on failure
if: failure()

613
.greptile/files.json
View file

@ -1,613 +0,0 @@
{
"files": [
{
"path": "litellm/_v2/AGENTS.md",
"description": "Conventions for code under litellm/_v2/",
"scope": [
"litellm/_v2/**"
]
},
{
"path": "litellm/_v2/cache/AGENTS.md",
"description": "Conventions for code under litellm/_v2/cache/",
"scope": [
"litellm/_v2/cache/**"
]
},
{
"path": "litellm/proxy/_experimental/mcp_server/AGENTS.md",
"description": "Conventions for code under litellm/proxy/_experimental/mcp_server/",
"scope": [
"litellm/proxy/_experimental/mcp_server/**"
]
},
{
"path": "litellm/rust_bridge/AGENTS.md",
"description": "Conventions for code under litellm/rust_bridge/",
"scope": [
"litellm/rust_bridge/**"
]
},
{
"path": "litellm/tracing/AGENTS.md",
"description": "Conventions for code under litellm/tracing/",
"scope": [
"litellm/tracing/**"
]
},
{
"path": "litellm-rust/AGENTS.md",
"description": "Conventions for code under litellm-rust/",
"scope": [
"litellm-rust/**"
]
},
{
"path": "litellm-rust/crates/cache-response/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/cache-response/",
"scope": [
"litellm-rust/crates/cache-response/**"
]
},
{
"path": "litellm-rust/crates/callbacks-legacy-python/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/callbacks-legacy-python/",
"scope": [
"litellm-rust/crates/callbacks-legacy-python/**"
]
},
{
"path": "litellm-rust/crates/coroutine/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/coroutine/",
"scope": [
"litellm-rust/crates/coroutine/**"
]
},
{
"path": "litellm-rust/crates/gateway/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/gateway/",
"scope": [
"litellm-rust/crates/gateway/**"
]
},
{
"path": "litellm-rust/crates/gateway-auth/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/gateway-auth/",
"scope": [
"litellm-rust/crates/gateway-auth/**"
]
},
{
"path": "litellm-rust/crates/gateway-inference/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/gateway-inference/",
"scope": [
"litellm-rust/crates/gateway-inference/**"
]
},
{
"path": "litellm-rust/crates/gateway-mcp/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/gateway-mcp/",
"scope": [
"litellm-rust/crates/gateway-mcp/**"
]
},
{
"path": "litellm-rust/crates/host/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/host/",
"scope": [
"litellm-rust/crates/host/**"
]
},
{
"path": "litellm-rust/crates/host/src/machine/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/host/src/machine/",
"scope": [
"litellm-rust/crates/host/src/machine/**"
]
},
{
"path": "litellm-rust/crates/host-http/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/host-http/",
"scope": [
"litellm-rust/crates/host-http/**"
]
},
{
"path": "litellm-rust/crates/host-native/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/host-native/",
"scope": [
"litellm-rust/crates/host-native/**"
]
},
{
"path": "litellm-rust/crates/host-python/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/host-python/",
"scope": [
"litellm-rust/crates/host-python/**"
]
},
{
"path": "litellm-rust/crates/http/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/http/",
"scope": [
"litellm-rust/crates/http/**"
]
},
{
"path": "litellm-rust/crates/inference/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/inference/",
"scope": [
"litellm-rust/crates/inference/**"
]
},
{
"path": "litellm-rust/crates/inference-chat/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/inference-chat/",
"scope": [
"litellm-rust/crates/inference-chat/**"
]
},
{
"path": "litellm-rust/crates/inference-messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/inference-messages/",
"scope": [
"litellm-rust/crates/inference-messages/**"
]
},
{
"path": "litellm-rust/crates/inference-ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/inference-ocr/",
"scope": [
"litellm-rust/crates/inference-ocr/**"
]
},
{
"path": "litellm-rust/crates/inference-responses/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/inference-responses/",
"scope": [
"litellm-rust/crates/inference-responses/**"
]
},
{
"path": "litellm-rust/crates/inference-transcription/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/inference-transcription/",
"scope": [
"litellm-rust/crates/inference-transcription/**"
]
},
{
"path": "litellm-rust/crates/llms/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/",
"scope": [
"litellm-rust/crates/llms/**"
]
},
{
"path": "litellm-rust/crates/llms/src/anthropic/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/",
"scope": [
"litellm-rust/crates/llms/src/anthropic/**"
]
},
{
"path": "litellm-rust/crates/llms/src/anthropic/batches/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/batches/",
"scope": [
"litellm-rust/crates/llms/src/anthropic/batches/**"
]
},
{
"path": "litellm-rust/crates/llms/src/anthropic/chat/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/chat/",
"scope": [
"litellm-rust/crates/llms/src/anthropic/chat/**"
]
},
{
"path": "litellm-rust/crates/llms/src/anthropic/count_tokens/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/count_tokens/",
"scope": [
"litellm-rust/crates/llms/src/anthropic/count_tokens/**"
]
},
{
"path": "litellm-rust/crates/llms/src/anthropic/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/anthropic/messages/",
"scope": [
"litellm-rust/crates/llms/src/anthropic/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/aws_textract/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/aws_textract/ocr/",
"scope": [
"litellm-rust/crates/llms/src/aws_textract/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms/src/azure_ai/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/azure_ai/messages/",
"scope": [
"litellm-rust/crates/llms/src/azure_ai/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/azure_ai/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/azure_ai/ocr/",
"scope": [
"litellm-rust/crates/llms/src/azure_ai/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms/src/base_llm/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/base_llm/messages/",
"scope": [
"litellm-rust/crates/llms/src/base_llm/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/bedrock/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/",
"scope": [
"litellm-rust/crates/llms/src/bedrock/**"
]
},
{
"path": "litellm-rust/crates/llms/src/bedrock/audio_transcription/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/audio_transcription/",
"scope": [
"litellm-rust/crates/llms/src/bedrock/audio_transcription/**"
]
},
{
"path": "litellm-rust/crates/llms/src/bedrock/chat/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/chat/",
"scope": [
"litellm-rust/crates/llms/src/bedrock/chat/**"
]
},
{
"path": "litellm-rust/crates/llms/src/bedrock/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/bedrock/messages/",
"scope": [
"litellm-rust/crates/llms/src/bedrock/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/cohere/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/cohere/ocr/",
"scope": [
"litellm-rust/crates/llms/src/cohere/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms/src/deepseek/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/deepseek/messages/",
"scope": [
"litellm-rust/crates/llms/src/deepseek/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/mistral/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/mistral/ocr/",
"scope": [
"litellm-rust/crates/llms/src/mistral/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms/src/openai/responses/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/openai/responses/",
"scope": [
"litellm-rust/crates/llms/src/openai/responses/**"
]
},
{
"path": "litellm-rust/crates/llms/src/openai_like/chat/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/openai_like/chat/",
"scope": [
"litellm-rust/crates/llms/src/openai_like/chat/**"
]
},
{
"path": "litellm-rust/crates/llms/src/reducto/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/reducto/ocr/",
"scope": [
"litellm-rust/crates/llms/src/reducto/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms/src/vertex_ai/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/vertex_ai/messages/",
"scope": [
"litellm-rust/crates/llms/src/vertex_ai/messages/**"
]
},
{
"path": "litellm-rust/crates/llms/src/vertex_ai/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms/src/vertex_ai/ocr/",
"scope": [
"litellm-rust/crates/llms/src/vertex_ai/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms-types/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/",
"scope": [
"litellm-rust/crates/llms-types/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/formats/audio_transcription/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/audio_transcription/",
"scope": [
"litellm-rust/crates/llms-types/src/formats/audio_transcription/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/formats/batches/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/batches/",
"scope": [
"litellm-rust/crates/llms-types/src/formats/batches/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/formats/chat_completions/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/chat_completions/",
"scope": [
"litellm-rust/crates/llms-types/src/formats/chat_completions/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/messages/",
"scope": [
"litellm-rust/crates/llms-types/src/formats/messages/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/formats/ocr/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/ocr/",
"scope": [
"litellm-rust/crates/llms-types/src/formats/ocr/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/formats/responses/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/formats/responses/",
"scope": [
"litellm-rust/crates/llms-types/src/formats/responses/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/providers/anthropic/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/anthropic/",
"scope": [
"litellm-rust/crates/llms-types/src/providers/anthropic/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/providers/bedrock/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/bedrock/",
"scope": [
"litellm-rust/crates/llms-types/src/providers/bedrock/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/providers/minimax/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/minimax/",
"scope": [
"litellm-rust/crates/llms-types/src/providers/minimax/**"
]
},
{
"path": "litellm-rust/crates/llms-types/src/providers/vertex_ai/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/llms-types/src/providers/vertex_ai/",
"scope": [
"litellm-rust/crates/llms-types/src/providers/vertex_ai/**"
]
},
{
"path": "litellm-rust/crates/model-catalog/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/model-catalog/",
"scope": [
"litellm-rust/crates/model-catalog/**"
]
},
{
"path": "litellm-rust/crates/python-bridge/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/python-bridge/",
"scope": [
"litellm-rust/crates/python-bridge/**"
]
},
{
"path": "litellm-rust/crates/python-bridge/src/cache/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/python-bridge/src/cache/",
"scope": [
"litellm-rust/crates/python-bridge/src/cache/**"
]
},
{
"path": "litellm-rust/crates/python-bridge/src/cache/native/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/python-bridge/src/cache/native/",
"scope": [
"litellm-rust/crates/python-bridge/src/cache/native/**"
]
},
{
"path": "litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/python-bridge/src/cache/python/",
"scope": [
"litellm-rust/crates/python-bridge/src/cache/python/**"
]
},
{
"path": "litellm-rust/crates/python-bridge/src/routes/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/python-bridge/src/routes/",
"scope": [
"litellm-rust/crates/python-bridge/src/routes/**"
]
},
{
"path": "litellm-rust/crates/python-compat/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/python-compat/",
"scope": [
"litellm-rust/crates/python-compat/**"
]
},
{
"path": "litellm-rust/crates/secrets-aws/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/secrets-aws/",
"scope": [
"litellm-rust/crates/secrets-aws/**"
]
},
{
"path": "litellm-rust/crates/secrets-azure/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/secrets-azure/",
"scope": [
"litellm-rust/crates/secrets-azure/**"
]
},
{
"path": "litellm-rust/crates/secrets-cyberark/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/secrets-cyberark/",
"scope": [
"litellm-rust/crates/secrets-cyberark/**"
]
},
{
"path": "litellm-rust/crates/secrets-google/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/secrets-google/",
"scope": [
"litellm-rust/crates/secrets-google/**"
]
},
{
"path": "litellm-rust/crates/secrets-hashicorp/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/secrets-hashicorp/",
"scope": [
"litellm-rust/crates/secrets-hashicorp/**"
]
},
{
"path": "litellm-rust/crates/storage-clickhouse/AGENTS.md",
"description": "Conventions for code under litellm-rust/crates/storage-clickhouse/",
"scope": [
"litellm-rust/crates/storage-clickhouse/**"
]
},
{
"path": "litellm-rust/docs/adrs/AGENTS.md",
"description": "Conventions for code under litellm-rust/docs/adrs/",
"scope": [
"litellm-rust/docs/adrs/**"
]
},
{
"path": "tests/AGENTS.md",
"description": "Conventions for code under tests/",
"scope": [
"tests/**"
]
},
{
"path": "tests/e2e/AGENTS.md",
"description": "Conventions for code under tests/e2e/",
"scope": [
"tests/e2e/**"
]
},
{
"path": "tests/e2e_harness/AGENTS.md",
"description": "Conventions for code under tests/e2e_harness/",
"scope": [
"tests/e2e_harness/**"
]
},
{
"path": "tests/integration/AGENTS.md",
"description": "Conventions for code under tests/integration/",
"scope": [
"tests/integration/**"
]
},
{
"path": "tests/integration/translation/AGENTS.md",
"description": "Conventions for code under tests/integration/translation/",
"scope": [
"tests/integration/translation/**"
]
},
{
"path": "tests/rust-python-harness/AGENTS.md",
"description": "Conventions for code under tests/rust-python-harness/",
"scope": [
"tests/rust-python-harness/**"
]
},
{
"path": "tests/rust-python-harness/strategies/e2e_parity/AGENTS.md",
"description": "Conventions for code under tests/rust-python-harness/strategies/e2e_parity/",
"scope": [
"tests/rust-python-harness/strategies/e2e_parity/**"
]
},
{
"path": "tests/rust-python-harness/strategies/trace_parity/AGENTS.md",
"description": "Conventions for code under tests/rust-python-harness/strategies/trace_parity/",
"scope": [
"tests/rust-python-harness/strategies/trace_parity/**"
]
},
{
"path": "tests/rust-python-harness/strategies/unit_tests_parity/AGENTS.md",
"description": "Conventions for code under tests/rust-python-harness/strategies/unit_tests_parity/",
"scope": [
"tests/rust-python-harness/strategies/unit_tests_parity/**"
]
},
{
"path": "tests/rust-python-harness/strategies/unit_tests_rust/AGENTS.md",
"description": "Conventions for code under tests/rust-python-harness/strategies/unit_tests_rust/",
"scope": [
"tests/rust-python-harness/strategies/unit_tests_rust/**"
]
},
{
"path": "tests/test_litellm_rust/AGENTS.md",
"description": "Conventions for code under tests/test_litellm_rust/",
"scope": [
"tests/test_litellm_rust/**"
]
},
{
"path": "tests/unit/AGENTS.md",
"description": "Conventions for code under tests/unit/",
"scope": [
"tests/unit/**"
]
},
{
"path": "tests/unit/rust_bridge/AGENTS.md",
"description": "Conventions for code under tests/unit/rust_bridge/",
"scope": [
"tests/unit/rust_bridge/**"
]
},
{
"path": "ui/litellm-dashboard/AGENTS.md",
"description": "Conventions for code under ui/litellm-dashboard/",
"scope": [
"ui/litellm-dashboard/**"
]
},
{
"path": "ui/litellm-dashboard/src/components/chat/AGENTS.md",
"description": "Conventions for code under ui/litellm-dashboard/src/components/chat/",
"scope": [
"ui/litellm-dashboard/src/components/chat/**"
]
}
]
}

2
AGENTS.md
View file

@ -130,4 +130,4 @@ Before implementing:
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify
Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review. Never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or raise `ANY_CAPS` to satisfy a check
Before requesting maintainer review, verify the current PR tip passes required CI and code coverage and has acceptable reviews from the review bots on the PR, such as Veria and Bugbot. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review. Never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or raise `ANY_CAPS` to satisfy a check

7
README.md
View file

@ -9,7 +9,7 @@
<a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render" height="40"></a>
<a href="https://railway.com/deploy/RhvhdC?referralCode=7mRv9K&utm_medium=integration&utm_source=template&utm_campaign=generic"><img src="https://railway.com/button.svg" alt="Deploy on Railway" height="40"></a>
<a href="https://console.aws.amazon.com/cloudshell/home" target="_blank" rel="nofollow"><img src="./.github/deploy-on-aws.png" alt="Deploy on AWS" height="40"></a>
<a href="https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true" target="_blank" rel="nofollow"><img src="./.github/deploy-on-gcp.png" alt="Deploy on GCP" height="40"></a>
<a href="https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Fterraform-google-litellm&cloudshell_git_branch=v1.105.0&cloudshell_workspace=examples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true" target="_blank" rel="nofollow"><img src="./.github/deploy-on-gcp.png" alt="Deploy on GCP" height="40"></a>
</p>
</p>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://litellm.ai/enterprise"target="_blank">Enterprise Tier</a> | <a href="https://www.litellm.ai/ai-gateway" target="_blank">Website</a></h4>
@ -342,6 +342,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Cohere Chat (`cohere_chat`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | | | | | | | |
| [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | | | | | | | |
| [CoralBricks (`coralbricks`)](https://docs.litellm.ai/docs/providers/coralbricks) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
@ -526,9 +527,9 @@ Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra
#### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB
[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.png)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true)
[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.png)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Fterraform-google-litellm&cloudshell_git_branch=v1.105.0&cloudshell_workspace=examples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true)
Real 1-click. Opens Cloud Shell, clones this repo, and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md) — pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply.
Real 1-click. Opens Cloud Shell, clones the [terraform-google-litellm](https://github.com/BerriAI/terraform-google-litellm) module mirror at the latest stable release (a small download instead of the full litellm repo), and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md). Pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply
[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/google/latest)

4
ci_cd/cost_map_guard.py
View file

@ -56,7 +56,7 @@ def _rendered_schema(cost_map: CostMap) -> str:
def _evaluation_output_price_failure(
key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
) -> str | None:
if not isinstance(entry, dict) or entry.get("mode") != "evaluation":
if not isinstance(entry, dict) or entry.get("mode") not in ("evaluation", "decisions"):
return None
output_cost: Final = entry.get("output_cost_per_token")
if (
@ -67,7 +67,7 @@ def _evaluation_output_price_failure(
):
return None
return (
f"{key}: mode evaluation rows bill input only; output_cost_per_token must be 0 unless the vendor "
f"{key}: mode evaluation and decisions rows bill input only; output_cost_per_token must be 0 unless the vendor "
"prices output tokens for this model (add the source to EVALUATION_OUTPUT_PRICE_SOURCES)"
)

2
db_scripts/partition_spend_logs.sql
View file

@ -55,6 +55,8 @@ ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_session_id_idx"
RENAME TO "LiteLLM_SpendLogs_legacy_session_id_idx";
ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx"
RENAME TO "LiteLLM_SpendLogs_legacy_api_key_startTime_idx";
ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_litellm_call_id_idx"
RENAME TO "LiteLLM_SpendLogs_legacy_litellm_call_id_idx";
CREATE TABLE "LiteLLM_SpendLogs" (
LIKE "LiteLLM_SpendLogs_legacy" INCLUDING DEFAULTS INCLUDING GENERATED

14
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -1162,7 +1162,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
return False
async def check_file_ids_access(self, file_ids: List[str], user_api_key_dict: UserAPIKeyAuth) -> None:
async def check_file_ids_access(self, file_ids: Sequence[str], user_api_key_dict: UserAPIKeyAuth) -> None:
"""
Check if the user has access to a list of file IDs.
Only checks managed (unified) file IDs.
@ -1269,6 +1269,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
elif call_type == CallTypes.acreate_batch.value:
input_file_id = cast(Optional[str], data.get("input_file_id"))
if input_file_id:
await self.check_file_ids_access((input_file_id,), user_api_key_dict)
model_file_id_mapping = await self.get_model_file_id_mapping(
[input_file_id], user_api_key_dict.parent_otel_span
)
@ -1315,11 +1316,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
elif retrieve_object_id and accessor_key == "batch_id":
await self.enforce_batch_object_access(retrieve_object_id, user_api_key_dict)
elif call_type == CallTypes.acreate_fine_tuning_job.value:
input_file_id = cast(Optional[str], data.get("training_file"))
if input_file_id:
model_file_id_mapping = await self.get_model_file_id_mapping(
[input_file_id], user_api_key_dict.parent_otel_span
)
fine_tuning_file_ids: Final = tuple(
file_id
for file_id in (data.get("training_file"), data.get("validation_file"))
if isinstance(file_id, str) and file_id
)
await self.check_file_ids_access(fine_tuning_file_ids, user_api_key_dict)
return data

3
litellm-rust/Cargo.lock generated
View file

@ -4024,6 +4024,7 @@ dependencies = [
"litellm-cache-memory",
"litellm-cache-response",
"litellm-core-utils",
"litellm-framer",
"litellm-host",
"litellm-host-native",
"litellm-http",
@ -4107,9 +4108,11 @@ name = "litellm-inference-testing"
version = "0.1.0"
dependencies = [
"futures-util",
"litellm-auth",
"litellm-http",
"litellm-inference",
"litellm-secrets",
"tokio",
]
[[package]]

83
litellm-rust/crates/callbacks-legacy-python/src/adapter.rs
View file

@ -442,7 +442,7 @@ impl LegacyLogging {
py: Python<'_>,
facts: &litellm_host::interceptors::ExecutionFacts,
) -> PyResult<HookStep<Self, ()>> {
use litellm_host::interceptors::ResultSource;
use litellm_host::interceptors::{Cost, ResultSource};
let logger = self.logger()?.object(py);
let params = logger
@ -462,6 +462,12 @@ impl LegacyLogging {
),
)?;
let details = logger.getattr("model_call_details")?;
let reported_cost = match &facts.cost {
Cost::Reported { amount } => amount.as_f64(),
Cost::CacheHit | Cost::Deferred => None,
};
logger.setattr("_provider_reported_cost", reported_cost)?;
details.call_method1("pop", ("response_cost", py.None()))?;
self.cache_key = match &facts.source {
ResultSource::Provider => None,
ResultSource::Cache { key } => Some(key.clone()),
@ -958,6 +964,81 @@ on_pre_call = lambda additional_args: None
check = lambda: None
";
#[rstest]
#[case::reported(Some(0.37), false)]
#[case::free(Some(0.0), false)]
#[case::deferred(None, false)]
#[case::cached(None, true)]
fn result_ready_hands_only_a_reported_cost_to_the_spend_logging_record(
#[case] reported: Option<f64>,
#[case] cache_hit: bool,
#[values(0.001, 7.0)] existing_cost: f64,
) {
Python::initialize();
Python::attach(|py| {
let locals = namespace(py, PAYLOAD_LOGGER);
run(
py,
&locals,
c"
logger.litellm_params = {}
logger.optional_params = {}
logger.model_call_details = {'response_cost': 7.0}
logger._provider_reported_cost = None
",
);
let mut logging = LegacyLogging {
logger: Some(PythonLogger::new(local(&locals, "logger").unbind())),
..legacy_call(py, &locals, false)
};
let source = if cache_hit {
litellm_host::interceptors::ResultSource::Cache {
key: "cached-test".into(),
}
} else {
litellm_host::interceptors::ResultSource::Provider
};
let facts = litellm_host::interceptors::ExecutionFacts {
provider: litellm_host::interceptors::ProviderIdentity {
model: "native-test".into(),
provider: "edenai".into(),
},
cost: litellm_host::interceptors::Cost::settle(
&source,
reported.and_then(serde_json::Number::from_f64),
),
source,
};
let previous = litellm_host::interceptors::ExecutionFacts {
provider: facts.provider.clone(),
cost: litellm_host::interceptors::Cost::Reported {
amount: serde_json::Number::from_f64(9.0).unwrap(),
},
source: litellm_host::interceptors::ResultSource::Provider,
};
logging.result_ready(py, &previous).unwrap();
local(&locals, "logger")
.getattr("model_call_details")
.unwrap()
.set_item("response_cost", existing_cost)
.unwrap();
let step = logging.result_ready(py, &facts).unwrap();
assert!(matches!(step, HookStep::Ready(())));
locals.set_item("reported", reported).unwrap();
locals.set_item("cache_hit", cache_hit).unwrap();
run(
py,
&locals,
c"
assert logger.model_call_details.get('response_cost') is None, logger.model_call_details
assert logger._provider_reported_cost == reported, logger._provider_reported_cost
assert logger.model_call_details['cache_hit'] is cache_hit, logger.model_call_details
assert logger.update['custom_llm_provider'] == 'edenai', logger.update
",
);
});
}
const DOCUMENT: &str = "data:application/pdf;base64,YWJj";
const EDITED: &str = "data:application/pdf;base64,ZWRpdGVk";

4
litellm-rust/crates/callbacks-legacy-python/src/python.rs
View file

@ -1,7 +1,7 @@
use pyo3::prelude::*;
use strum::{IntoStaticStr, VariantArray};
const MODULE: &str = "litellm.rust_bridge.callbacks_legacy_python";
const MODULE: &str = "litellm.rust_bridge.host.callbacks_legacy_python";
/// Every litellm Python internal the native call still borrows, grouped by the subsystem it
/// belongs to. Rust drives the call; these exist only so behaviour that Python owns today
@ -9,7 +9,7 @@ const MODULE: &str = "litellm.rust_bridge.callbacks_legacy_python";
/// A group is deleted once Rust owns that subsystem, so this enum only shrinks. Calling a
/// user's own callback is not borrowing and does not belong here.
///
/// `litellm/rust_bridge/callbacks_legacy_python.py` is the only Python module behind it, and
/// `litellm/rust_bridge/host/callbacks_legacy_python.py` is the only Python module behind it, and
/// `python_contract.json` pins each function's parameters on both sides.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum LegacyPython {

6
litellm-rust/crates/callbacks-legacy-python/src/test_support.rs
View file

@ -6,7 +6,7 @@ use pyo3::types::{PyDict, PyTuple};
use crate::{LegacyLogging, PublicCall};
/// The parameters of every `callbacks_legacy_python` function, as the real module declares them.
/// `tests/unit/rust_bridge/test_callbacks_legacy_python.py` pins this file to the Python
/// `tests/unit/rust_bridge/host/test_callbacks_legacy_python.py` pins this file to the Python
/// signatures, and [`namespace`] binds every fake call against it.
pub(crate) const PYTHON_CONTRACT: &str = include_str!("../python_contract.json");
@ -23,10 +23,10 @@ import sys
import traceback
import types
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.callbacks_legacy_python'):
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host', 'litellm.rust_bridge.host.callbacks_legacy_python'):
sys.modules.setdefault(name, types.ModuleType(name))
legacy = sys.modules['litellm.rust_bridge.callbacks_legacy_python']
legacy = sys.modules['litellm.rust_bridge.host.callbacks_legacy_python']
CONTRACT = json.loads(python_contract)

10
litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs
View file

@ -16,18 +16,28 @@ pub enum LlmProviders {
AzureAi,
#[strum(serialize = "bedrock")]
Bedrock,
#[strum(serialize = "bedrock_mantle")]
BedrockMantle,
#[strum(serialize = "cohere")]
Cohere,
#[strum(serialize = "deepseek")]
Deepseek,
#[strum(serialize = "edenai")]
Edenai,
#[strum(serialize = "minimax")]
Minimax,
#[strum(serialize = "mistral")]
Mistral,
#[strum(serialize = "openai")]
Openai,
#[strum(serialize = "openai_like")]
OpenaiLike,
#[strum(serialize = "openrouter")]
Openrouter,
#[strum(serialize = "reducto")]
Reducto,
#[strum(serialize = "tencent")]
Tencent,
#[strum(serialize = "vertex_ai")]
VertexAi,
}

38
litellm-rust/crates/gateway-inference/src/messages.rs
View file

@ -19,7 +19,12 @@ use crate::{Deployment, Error, Gateway, JsonObject, RequestId, request};
/// Client headers Python forwards to Anthropic-speaking providers on every call.
const ANTHROPIC_API_HEADERS: [&str; 2] = ["anthropic-version", "anthropic-beta"];
const ANTHROPIC_API_HEADER_PROVIDERS: &str = "anthropic,bedrock,bedrock_mantle,vertex_ai";
const OPENROUTER_API_HEADERS: [&str; 2] = [
"x-openrouter-metadata",
"x-openrouter-experimental-metadata",
];
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter";
pub async fn create(
State(gateway): State<Arc<Gateway>>,
@ -88,17 +93,28 @@ fn project(
}
fn anthropic_api_headers(headers: &HeaderMap) -> Option<ProviderSpecificHeaders> {
let extra_headers: Map<String, Value> = ANTHROPIC_API_HEADERS
let scopes = [
(
ANTHROPIC_API_HEADER_PROVIDERS,
ANTHROPIC_API_HEADERS.as_slice(),
),
("openrouter", OPENROUTER_API_HEADERS.as_slice()),
];
let forwarded: Vec<ProviderSpecificHeader> = scopes
.into_iter()
.filter_map(|name| {
let value = headers.get(name)?.to_str().ok()?;
Some((name.to_owned(), Value::from(value)))
.filter_map(|(provider, names)| {
let extra_headers: Map<String, Value> = names
.iter()
.filter_map(|name| {
let value = headers.get(*name)?.to_str().ok()?;
Some(((*name).to_owned(), Value::from(value)))
})
.collect();
(!extra_headers.is_empty()).then(|| ProviderSpecificHeader {
custom_llm_provider: provider.into(),
extra_headers,
})
})
.collect();
(!extra_headers.is_empty()).then(|| {
ProviderSpecificHeaders::One(ProviderSpecificHeader {
custom_llm_provider: ANTHROPIC_API_HEADER_PROVIDERS.into(),
extra_headers,
})
})
(!forwarded.is_empty()).then_some(ProviderSpecificHeaders::Many(forwarded))
}

49
litellm-rust/crates/gateway-inference/tests/messages.rs
View file

@ -192,3 +192,52 @@ async fn model_permissions_prevent_provider_calls(#[case] path: &str) {
assert_eq!(response.status(), 403);
assert!(upstream.received_requests().await.unwrap().is_empty());
}
#[rstest]
#[case::completion(false)]
#[case::streaming(true)]
#[tokio::test]
async fn openrouter_receives_native_messages_headers_and_its_deployment_bearer(
#[case] streaming: bool,
) {
let upstream = MockServer::start().await;
let message = json!({"id": "msg_test", "type": "message", "role": "assistant", "model": "anthropic/test-model", "content": [{"type": "text", "text": "hello"}], "stop_reason": "end_turn", "stop_sequence": null, "usage": {"input_tokens": 1, "output_tokens": 1, "cost": 0.001, "cost_details": "unexpected"}, "openrouter_metadata": {"provider": "test-provider"}});
let sse = "event: message_delta\ndata: {\"type\":\"message_delta\",\"usage\":{\"cost\":0.001}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n";
let template = if streaming {
ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")
} else {
ResponseTemplate::new(200).set_body_json(message.clone())
};
Mock::given(method("POST"))
.and(path("/api/v1/messages"))
.and(header("authorization", "Bearer test-key"))
.and(header("anthropic-beta", "test-feature"))
.and(header("anthropic-version", "test-version"))
.and(header("x-openrouter-metadata", "enabled"))
.and(body_json(json!({"model": "anthropic/test-model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16, "stream": streaming, "provider": {"order": ["test-provider"]}})))
.respond_with(template).expect(1).mount(&upstream).await;
let request = Request::post("/v1/messages")
.header("content-type", "application/json")
.header("anthropic-beta", "test-feature")
.header("anthropic-version", "test-version")
.header("x-openrouter-metadata", "enabled")
.header("x-api-key", "proxy-key")
.header("authorization", "Bearer proxy-key")
.body(Body::from(json!({"model": "public/model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16, "stream": streaming, "provider": {"order": ["test-provider"]}}).to_string())).unwrap();
let response = support::app(
"openrouter/anthropic/test-model",
&format!("{}/api/v1", upstream.uri()),
)
.oneshot(request)
.await
.unwrap();
assert_eq!(response.status(), 200);
if streaming {
assert_eq!(to_bytes(response.into_body(), 4096).await.unwrap(), sse);
} else {
assert_eq!(support::json(response).await, message);
}
let requests = upstream.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
assert!(!requests[0].headers.contains_key("x-api-key"));
}

2
litellm-rust/crates/gateway-mcp/AGENTS.md
View file

@ -24,7 +24,7 @@ From the repository root, this loopback-only example runs the existing Python MC
```sh
cargo run --manifest-path litellm-rust/Cargo.toml -p litellm-gateway-mcp --example stdio_proxy -- \
.venv/bin/python -c 'import runpy; runpy.run_path("tests/mcp_tests/mcp_e2e_upstream_server.py")["mcp"].run(transport="stdio")'
.venv/bin/python -c 'import runpy; runpy.run_path("tests/integration/_support/mcp_e2e_upstream_server.py")["mcp"].run(transport="stdio")'
```
```sh

2
litellm-rust/crates/gateway/README.md
View file

@ -7,7 +7,7 @@ The binary reads YAML from `LITELLM_CONFIG` and mounts configured MCP servers al
From the repository root, start the Python fixture in one terminal:
```sh
MCP_HOST=127.0.0.1 MCP_PORT=8090 .venv/bin/python tests/mcp_tests/mcp_e2e_upstream_server.py
MCP_HOST=127.0.0.1 MCP_PORT=8090 .venv/bin/python tests/integration/_support/mcp_e2e_upstream_server.py
```
Start the gateway in another terminal with the example configuration:

22
litellm-rust/crates/host/src/interceptors.rs
View file

@ -42,10 +42,32 @@ pub enum ResultSource {
Cache { key: String },
}
/// What the call cost, settled once by the route. A host records it; it never re-derives it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Cost {
/// The amount the provider itself billed, taken from its response.
Reported { amount: serde_json::Number },
/// Served from the cache, so nothing was billed.
CacheHit,
/// No provider figure: the host estimates from usage and its price table.
Deferred,
}
impl Cost {
pub fn settle(source: &ResultSource, reported: Option<serde_json::Number>) -> Self {
match (source, reported) {
(ResultSource::Cache { .. }, _) => Self::CacheHit,
(ResultSource::Provider, Some(amount)) => Self::Reported { amount },
(ResultSource::Provider, None) => Self::Deferred,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ExecutionFacts {
pub provider: ProviderIdentity,
pub source: ResultSource,
pub cost: Cost,
}
pub trait Interceptors<E>: Send + Sync {

44
litellm-rust/crates/http/src/request.rs
View file

@ -138,7 +138,9 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool {
}
let value = value.trim();
value.len() > 7
&& value[..7].eq_ignore_ascii_case("bearer ")
&& value
.get(..7)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("bearer "))
&& !value[7..].trim().is_empty()
})
}
@ -279,31 +281,19 @@ mod tests {
assert!(!has_header(&headers, "authorization"));
}
#[test]
fn bearer_detection_requires_a_non_empty_token() {
assert!(has_bearer_auth(&[(
"Authorization".to_string(),
"Bearer abc".to_string()
)]));
assert!(has_bearer_auth(&[(
"authorization".to_string(),
"bearer abc".to_string()
)]));
assert!(!has_bearer_auth(&[(
"Authorization".to_string(),
"Bearer ".to_string()
)]));
assert!(!has_bearer_auth(&[(
"authorization".to_string(),
String::new()
)]));
assert!(!has_bearer_auth(&[(
"Authorization".to_string(),
"Basic abc".to_string()
)]));
assert!(!has_bearer_auth(&[(
"x-api-key".to_string(),
"abc".to_string()
)]));
#[rstest::rstest]
#[case::bearer("Authorization", "Bearer abc", true)]
#[case::lowercase("authorization", "bearer abc", true)]
#[case::blank_token("Authorization", "Bearer ", false)]
#[case::empty("authorization", "", false)]
#[case::basic("Authorization", "Basic abc", false)]
#[case::different_header("x-api-key", "abc", false)]
#[case::non_ascii("authorization", "invali\u{00e9}auth", false)]
fn bearer_detection_requires_a_non_empty_token(
#[case] name: &str,
#[case] value: &str,
#[case] expected: bool,
) {
assert_eq!(has_bearer_auth(&[(name.into(), value.into())]), expected);
}
}

1
litellm-rust/crates/inference-messages/Cargo.toml
View file

@ -26,6 +26,7 @@ serde_json = { workspace = true, features = ["preserve_order"] }
tracing.workspace = true
[dev-dependencies]
litellm-framer = { workspace = true, features = ["sse"] }
litellm-cache-memory.workspace = true
litellm-host-native.workspace = true
litellm-http = { workspace = true, features = ["test-support"] }

21
litellm-rust/crates/inference-messages/src/common_utils.rs
View file

@ -1,14 +1,21 @@
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
use litellm_http::request::string_headers as shared_string_headers;
pub(super) use litellm_http::request::truncate_error_body;
use litellm_llms::tencent::messages::transformation::TENCENT_MESSAGES_CONFIG;
use litellm_llms::{
anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG,
azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG,
base_llm::messages::transformation::BaseMessagesConfig,
bedrock::messages::invoke_transformations::anthropic_claude3_transformation::BEDROCK_ANTHROPIC_MESSAGES_CONFIG,
deepseek::messages::transformation::DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG,
edenai::messages::transformation::EDENAI_MESSAGES_CONFIG,
minimax::messages::transformation::MINIMAX_MESSAGES_CONFIG,
vertex_ai::messages::transformation::VERTEX_ANTHROPIC_MESSAGES_CONFIG,
};
use litellm_llms::{
bedrock::claude_platform::messages::transformation::BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG,
bedrock_mantle::messages::transformation::BEDROCK_MANTLE_MESSAGES_CONFIG,
};
use serde_json::{Map, Value};
use super::Error;
@ -34,14 +41,25 @@ impl MessagesProvider {
/// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude
/// partner models on this route.
pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option<MessagesProvider> {
use litellm_llms::openrouter::messages::transformation::OPENROUTER_MESSAGES_CONFIG;
let config: &'static dyn BaseMessagesConfig = match provider {
LlmProviders::Tencent => &TENCENT_MESSAGES_CONFIG,
LlmProviders::Anthropic => &ANTHROPIC_MESSAGES_CONFIG,
LlmProviders::AzureAi => &AZURE_ANTHROPIC_MESSAGES_CONFIG,
LlmProviders::Bedrock if model.starts_with("claude_platform/") => {
&BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
}
LlmProviders::Bedrock if model.starts_with("mantle/") => &BEDROCK_MANTLE_MESSAGES_CONFIG,
LlmProviders::Bedrock => &BEDROCK_ANTHROPIC_MESSAGES_CONFIG,
LlmProviders::BedrockMantle => &BEDROCK_MANTLE_MESSAGES_CONFIG,
LlmProviders::Deepseek => &DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG,
LlmProviders::Edenai => &EDENAI_MESSAGES_CONFIG,
LlmProviders::Minimax => &MINIMAX_MESSAGES_CONFIG,
LlmProviders::VertexAi if model.to_ascii_lowercase().contains("claude") => {
&VERTEX_ANTHROPIC_MESSAGES_CONFIG
}
LlmProviders::Openrouter => &OPENROUTER_MESSAGES_CONFIG,
_ => return None,
};
Some(MessagesProvider { provider, config })
@ -64,10 +82,13 @@ mod tests {
use litellm_core_utils::get_llm_provider_logic::LlmProviders;
#[rstest]
#[case::tencent("tencent")]
#[case::anthropic("anthropic")]
#[case::azure_ai("azure_ai")]
#[case::bedrock("bedrock")]
#[case::deepseek("deepseek")]
#[case::edenai("edenai")]
#[case::minimax("minimax")]
#[case::vertex_ai("vertex_ai")]
fn provider_keeps_its_python_name(#[case] name: &str) {
let provider =

2
litellm-rust/crates/inference-messages/src/handler.rs
View file

@ -25,7 +25,7 @@ use litellm_inference::{context::CallContext, outbound::outbound_request};
pub(super) struct ProviderCall {
pub identity: ProviderIdentity,
pub wire: WireRequest,
provider: super::common_utils::MessagesProvider,
pub(super) provider: super::common_utils::MessagesProvider,
signer: Option<litellm_auth_aws::SigV4Signer>,
timeout: Option<Duration>,
stream: bool,

8
litellm-rust/crates/inference-messages/src/lib.rs
View file

@ -7,7 +7,7 @@ mod types;
use futures_util::FutureExt;
use litellm_auth::AuthServices;
use litellm_host::interceptors::{ExecutionFacts, Interceptors, ResultSource};
use litellm_host::interceptors::{Cost, ExecutionFacts, Interceptors, ResultSource};
use litellm_inference::{caching::CallCache, context::CallContext};
use litellm_secrets::source::SecretSource;
@ -87,6 +87,7 @@ impl MessagesRoute {
&request.wire,
);
let identity = request.identity.clone();
let config = request.provider.config();
let (output, source) = match cache.lookup().await {
Some(hit) => hit,
None => (
@ -94,9 +95,14 @@ impl MessagesRoute {
ResultSource::Provider,
),
};
let reported = match &output {
MessagesCallResponse::Complete(response) => config.reported_cost(response),
MessagesCallResponse::Stream { .. } => None,
};
context
.result_ready(ExecutionFacts {
provider: identity,
cost: Cost::settle(&source, reported),
source: source.clone(),
})
.await?;

61
litellm-rust/crates/inference-messages/src/prepare.rs
View file

@ -76,6 +76,10 @@ fn prepare_provider_request(
shaping,
..
} = call;
let connection_params = litellm_router_types::LitellmParams {
api_base: api_base.clone().or(litellm_params.api_base.clone()),
..litellm_params
};
let config = provider.config();
let env_lookup = |key: &str| secrets.get(key);
@ -95,25 +99,22 @@ fn prepare_provider_request(
let forwarded = string_headers(Some(
extra_headers.into_iter().flatten().chain(scoped).collect(),
))?;
let validated = config.validate_environment(
let forwarded = config.request_headers(
with_default_headers(forwarded, config.default_headers()),
&transformed,
);
let environment = config.validate_environment(
forwarded,
api_key.as_deref(),
&transformed.model,
&litellm_params,
&connection_params,
&env_lookup,
)?;
let environment = ValidatedEnvironment {
headers: config.request_headers(
with_default_headers(validated.headers, config.default_headers()),
&transformed,
),
auth: validated.auth,
};
let url = config.get_complete_url(
api_base.as_deref(),
&transformed.model,
&litellm_params,
&connection_params,
transformed.params.stream == Some(true),
&env_lookup,
)?;
@ -475,4 +476,44 @@ mod tests {
))
);
}
#[rstest]
#[case::mantle(
"bedrock_mantle/claude-test",
"https://bedrock-mantle.us-west-2.api.aws",
"/anthropic/v1/messages",
"bedrock"
)]
#[case::platform(
"bedrock/claude_platform/claude-test",
"https://aws-external-anthropic.us-west-2.api.aws",
"/v1/messages",
"aws-external-anthropic"
)]
fn explicit_endpoint_region_is_used_by_the_signer(
shaping: MessagesShaping,
#[case] model: &str,
#[case] base: &str,
#[case] path: &str,
#[case] service: &str,
) {
let prepared = prepare_with_secrets(MessagesCall {
body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
api_key: None,
api_base: Some(base.into()),
custom_llm_provider: None,
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: None,
shaping,
}, &|name: &str| match name {
"AWS_REGION_NAME" => Some("us-east-1".into()),
"ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
_ => None,
}).unwrap();
assert_eq!(prepared.url, format!("{base}{path}"));
assert!(
matches!(prepared.environment.auth, litellm_llms::base_llm::auth::AuthScheme::AwsSigV4 { region, service: actual, .. } if region == "us-west-2" && actual == service)
);
}
}

323
litellm-rust/crates/inference-messages/tests/bedrock_native/main.rs Normal file
View file

@ -0,0 +1,323 @@
use std::{sync::Arc, time::Duration};
use futures_util::TryStreamExt;
use litellm_inference_messages::{Error, MessagesCall, MessagesCallResponse, MessagesShaping};
use litellm_inference_testing::{RecordingSecrets, no_secrets};
use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
use rstest::{fixture, rstest};
use serde_json::{Value, json};
use wiremock::ResponseTemplate;
#[path = "../support/mod.rs"]
mod support;
use support::*;
const MODEL: &str = "claude-test";
#[fixture]
fn call() -> MessagesCall {
MessagesCall {
body: serde_json::from_value(json!({"model": MODEL, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})).unwrap(),
api_key: None,
api_base: None,
custom_llm_provider: None,
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: Some(Duration::from_secs(5)),
shaping: MessagesShaping::default(),
}
}
fn with_fields(call: MessagesCall, fields: Value) -> MessagesCall {
let body = serde_json::to_value(call.body).unwrap();
MessagesCall {
body: serde_json::from_value(Value::Object(
body.as_object()
.unwrap()
.clone()
.into_iter()
.chain(fields.as_object().unwrap().clone())
.collect(),
))
.unwrap(),
..call
}
}
fn with_model(call: MessagesCall, model: &str) -> MessagesCall {
MessagesCall {
body: MessagesRequest {
model: model.into(),
..call.body
},
..call
}
}
fn message_response() -> ResponseTemplate {
json_response(
json!({"id": "msg_1", "type": "message", "role": "assistant", "content": [{"type": "text", "text": "hi"}], "model": MODEL, "stop_reason": "end_turn", "usage": {"input_tokens": 1, "output_tokens": 2}}),
)
}
async fn run_with(
secrets: Arc<RecordingSecrets>,
call: MessagesCall,
) -> Result<MessagesCallResponse, Error> {
messages_route(secrets).execute(call, &(), None).await
}
async fn run(call: MessagesCall) -> Result<MessagesCallResponse, Error> {
run_with(Arc::new(RecordingSecrets::empty()), call).await
}
async fn run_message(call: MessagesCall) -> MessagesResponse {
match run(call).await.unwrap() {
MessagesCallResponse::Complete(message) => *message,
MessagesCallResponse::Stream { .. } => panic!("expected a completed message"),
}
}
#[rstest]
#[case::platform(
"bedrock/claude_platform/claude-test",
"/v1/messages",
"x-api-key",
"key"
)]
#[case::mantle(
"bedrock_mantle/anthropic.claude-test",
"/anthropic/v1/messages",
"authorization",
"Bearer key"
)]
#[case::mantle_alias(
"bedrock/mantle/anthropic.claude-test",
"/anthropic/v1/messages",
"authorization",
"Bearer key"
)]
#[tokio::test]
async fn prefixes_select_native_messages_and_strip_connection_fields(
call: MessagesCall,
#[case] model: &str,
#[case] path: &str,
#[case] header: &str,
#[case] credential: &str,
) {
let upstream = upstream([message_response()]).await;
let message = run_message(MessagesCall {
api_key: Some("key".into()),
api_base: Some(upstream.uri()),
custom_llm_provider: None,
..with_fields(
call,
json!({"model": model, "workspace_id": "request-workspace"}),
)
})
.await;
assert_eq!(message.model, MODEL);
let request = only_request(&upstream).await;
assert_eq!(request.url.path(), path);
assert_eq!(request.header(header), Some(credential));
assert_eq!(request.json()["model"], model.rsplit('/').next().unwrap());
if path == "/v1/messages" {
assert_eq!(
request.header("anthropic-workspace-id"),
Some("request-workspace")
);
assert_eq!(request.json().get("workspace_id"), None);
}
}
#[rstest]
#[case::platform(
"bedrock/claude_platform/claude-test",
"ANTHROPIC_AWS_API_KEY",
"ANTHROPIC_AWS_BASE_URL",
"/v1/messages",
"x-api-key",
"managed"
)]
#[case::mantle(
"bedrock_mantle/claude-test",
"AWS_BEARER_TOKEN_BEDROCK",
"BEDROCK_MANTLE_API_BASE",
"/anthropic/v1/messages",
"authorization",
"Bearer managed"
)]
#[tokio::test]
async fn managed_keys_bases_and_workspaces_reach_the_upstream(
call: MessagesCall,
#[case] model: &str,
#[case] key_name: &str,
#[case] base_name: &str,
#[case] path: &str,
#[case] header: &str,
#[case] credential: &str,
) {
let upstream = upstream([message_response()]).await;
let base = upstream.uri();
let secrets = Arc::new(RecordingSecrets::new([
(key_name, "managed"),
(base_name, base.as_str()),
("ANTHROPIC_AWS_WORKSPACE_ID", "managed-workspace"),
]));
let output = run_with(
secrets,
MessagesCall {
custom_llm_provider: None,
..with_model(call, model)
},
)
.await
.unwrap();
assert!(matches!(output, MessagesCallResponse::Complete(_)));
let request = only_request(&upstream).await;
assert_eq!(request.url.path(), path);
assert_eq!(request.header(header), Some(credential));
if path == "/v1/messages" {
assert_eq!(
request.header("anthropic-workspace-id"),
Some("managed-workspace")
);
}
}
#[rstest]
#[case::platform("bedrock/claude_platform/claude-test", "aws-external-anthropic")]
#[case::mantle("bedrock_mantle/claude-test", "bedrock")]
#[tokio::test]
async fn managed_aws_credentials_sign_the_exact_native_request(
call: MessagesCall,
#[case] model: &str,
#[case] service: &str,
) {
let upstream = upstream([message_response()]).await;
let secrets = Arc::new(RecordingSecrets::new([
("AWS_ACCESS_KEY_ID", "test-access"),
("AWS_SECRET_ACCESS_KEY", "test-secret"),
("AWS_SESSION_TOKEN", "test-session"),
("AWS_REGION_NAME", "us-west-2"),
("ANTHROPIC_AWS_WORKSPACE_ID", "managed-workspace"),
]));
run_with(
secrets,
MessagesCall {
api_base: Some(upstream.uri()),
custom_llm_provider: None,
..with_model(call, model)
},
)
.await
.unwrap();
let request = only_request(&upstream).await;
let auth = request.header("authorization").unwrap();
assert!(auth.starts_with("AWS4-HMAC-SHA256 Credential=test-access/"));
assert!(auth.contains(&format!("/us-west-2/{service}/aws4_request")));
assert_eq!(request.header("x-amz-security-token"), Some("test-session"));
assert_eq!(request.header("x-api-key"), None);
assert_eq!(request.json()["model"], "claude-test");
}
#[rstest]
#[tokio::test]
async fn private_mantle_endpoint_is_used_and_signed(call: MessagesCall) {
let upstream = upstream([message_response()]).await;
let params = litellm_router_types::LitellmParams {
aws: litellm_auth::AwsParams {
aws_bedrock_runtime_endpoint: Some(upstream.uri()),
aws_access_key_id: Some("private-access".into()),
aws_secret_access_key: Some("private-secret".into()),
aws_region_name: Some("us-west-2".into()),
..Default::default()
},
..Default::default()
};
run(MessagesCall {
custom_llm_provider: None,
litellm_params: params,
..with_model(call, "bedrock/mantle/claude-test")
})
.await
.unwrap();
let request = only_request(&upstream).await;
assert_eq!(request.url.path(), "/anthropic/v1/messages");
assert!(
request
.header("authorization")
.unwrap()
.contains("/us-west-2/bedrock/aws4_request")
);
}
#[rstest]
#[case::platform("bedrock/claude_platform/claude-test")]
#[case::mantle("bedrock_mantle/claude-test")]
#[tokio::test]
async fn native_streams_relay_every_sse_byte(call: MessagesCall, #[case] model: &str) {
let sse = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n";
let upstream =
upstream([ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")]).await;
let response = messages_route(no_secrets())
.execute(
MessagesCall {
api_key: Some("key".into()),
api_base: Some(upstream.uri()),
custom_llm_provider: None,
..with_fields(
call,
json!({"model": model, "stream": true, "workspace_id": "workspace"}),
)
},
&(),
None,
)
.await
.unwrap();
let MessagesCallResponse::Stream { chunks, .. } = response else {
panic!("expected SSE");
};
assert_eq!(
chunks.try_collect::<Vec<_>>().await.unwrap().concat(),
sse.as_bytes()
);
assert_eq!(only_request(&upstream).await.json()["stream"], true);
}
#[rstest]
#[case::missing(false)]
#[case::configured(true)]
#[tokio::test]
async fn deployment_workspace_is_required_and_sent_as_a_header(
call: MessagesCall,
#[case] configured: bool,
) {
let upstream = upstream([message_response()]).await;
let params = if configured {
serde_json::from_value(json!({"model": "bedrock/claude_platform/claude-test", "workspace_id": "deployment-workspace"})).unwrap()
} else {
Default::default()
};
let result = run(MessagesCall {
api_key: Some("key".into()),
api_base: Some(upstream.uri()),
litellm_params: params,
..with_model(call, "bedrock/claude_platform/claude-test")
})
.await;
if !configured {
assert!(matches!(result, Err(Error::Auth(_))));
assert!(received(&upstream).await.is_empty());
return;
}
assert!(matches!(result.unwrap(), MessagesCallResponse::Complete(_)));
let sent = only_request(&upstream).await;
assert_eq!(
sent.header("anthropic-workspace-id"),
Some("deployment-workspace")
);
assert_eq!(sent.json().get("workspace_id"), None);
}

56
litellm-rust/crates/inference-messages/tests/caching.rs
View file

@ -13,8 +13,8 @@ use litellm_cache_response::{
CacheOptions, CacheScope, ResponseCache, ResponseCacheConfig, ResponseCacheService, ScopedCache,
};
use litellm_host::interceptors::{
ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext, ResultSource,
WireRequest,
Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext,
ResultSource, WireRequest,
};
use litellm_inference::{
RouteError,
@ -241,3 +241,55 @@ async fn messages_cache_identity_follows_resolved_configuration_and_request_call
first.verify().await;
second.verify().await;
}
#[rstest]
#[case::reported(json!(0.37), Some(0.37))]
#[case::free(json!(0), Some(0.0))]
#[case::malformed(json!("invalid"), None)]
#[case::missing(Value::Null, None)]
#[tokio::test]
async fn provider_reported_cost_is_delivered_once_and_never_charged_for_a_cache_hit(
cache: Arc<dyn ResponseCacheService>,
#[case] cost: Value,
#[case] expected: Option<f64>,
) {
use litellm_inference_messages::{MessagesCall, MessagesCallResponse};
let response = json!({"id": "msg_cost", "type": "message", "role": "assistant", "model": "native-test",
"content": [{"type": "text", "text": "answer"}], "stop_reason": "end_turn", "stop_sequence": null,
"usage": {"input_tokens": 3, "output_tokens": 2}, "cost": cost});
let upstream = support::upstream([support::json_response(response)]).await;
let hooks = ChangingHooks::default();
let route = support::messages_route(litellm_inference_testing::no_secrets())
.with_cache(ScopedCache::new(cache, CacheScope::Shared));
let request = || {
MessagesCall {
body: serde_json::from_value(json!({"model": "native-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})).unwrap(),
api_key: Some("eden-key".into()), api_base: Some(upstream.uri()), custom_llm_provider: Some("edenai".into()),
litellm_params: Default::default(), extra_headers: None, provider_specific_header: None,
timeout: None, shaping: Default::default(),
}
};
let first = route.execute(request(), &hooks, None).await.unwrap();
let second = route.execute(request(), &hooks, None).await.unwrap();
let (MessagesCallResponse::Complete(first), MessagesCallResponse::Complete(second)) =
(first, second)
else {
panic!("expected Messages responses");
};
assert_eq!(first, second);
assert_eq!(first.extra.get("cost"), Some(&cost));
assert_eq!(support::received(&upstream).await.len(), 1);
let facts = hooks.facts.lock().unwrap();
let [provider, cached] = facts.as_slice() else {
panic!("expected provider and cache results");
};
assert_eq!(provider.source, ResultSource::Provider);
assert_eq!(
provider.cost,
expected.map_or(Cost::Deferred, |amount| Cost::Reported {
amount: serde_json::Number::from_f64(amount).unwrap()
})
);
assert!(matches!(cached.source, ResultSource::Cache { .. }));
assert_eq!(cached.cost, Cost::CacheHit);
}

8
litellm-rust/crates/inference-messages/tests/live.rs Normal file
View file

@ -0,0 +1,8 @@
#[path = "live/support.rs"]
mod support;
#[path = "live/edenai.rs"]
mod edenai;
#[path = "live/vertex_ai.rs"]
mod vertex_ai;

92
litellm-rust/crates/inference-messages/tests/live/edenai.rs Normal file
View file

@ -0,0 +1,92 @@
use litellm_host::interceptors::Cost;
use litellm_inference_messages::MessagesRoute;
use litellm_inference_testing::live::within_deadline;
use rstest::rstest;
use serde_json::{Number, Value, json};
use super::support::{LiveCall, assert_text, call, complete, request, route, stream_events};
const PROVIDER: &str = "edenai";
#[rstest]
#[case::basic(json!({"messages": [{"role": "user", "content": "Say hello."}]}))]
#[case::system_text(json!({
"system": "You are a helpful assistant.",
"messages": [{"role": "user", "content": "Say hello."}]
}))]
#[case::portable_cache_hint(json!({
"system": [{"type": "text", "text": "You are a helpful assistant.", "cache_control": {"type": "ephemeral", "ttl": "1h"}}],
"messages": [{"role": "user", "content": "Say hello."}]
}))]
#[ignore = "calls a real provider, requires credentials and a live model"]
#[tokio::test]
async fn completion_and_cost_handoff(route: MessagesRoute, #[case] payload: Value) {
within_deadline(async {
let host = LiveCall::new(PROVIDER);
let message = complete(request(&route, call(PROVIDER, payload), &host).await);
assert_text(&message);
host.assert_provider_result();
let expected_cost = message
.extra
.get("cost")
.and_then(|value| match value {
Value::Number(number) => number.as_f64(),
Value::String(text) => text.parse::<f64>().ok(),
_ => None,
})
.filter(|cost| *cost >= 0.0)
.and_then(Number::from_f64)
.map_or(Cost::Deferred, |amount| Cost::Reported { amount });
assert_eq!(host.facts().cost, expected_cost);
println!("{}", serde_json::to_string(&message).unwrap());
})
.await;
}
#[rstest]
#[ignore = "calls a real provider, requires credentials and a live model"]
#[tokio::test]
async fn streaming(route: MessagesRoute) {
within_deadline(async {
let host = LiveCall::new(PROVIDER);
let events = stream_events(
request(
&route,
call(
PROVIDER,
json!({
"stream": true, "messages": [{"role": "user", "content": "Say hello."}]
}),
),
&host,
)
.await,
)
.await;
assert!(events.iter().any(|event| {
event["delta"]["text"]
.as_str()
.is_some_and(|text| !text.is_empty())
}));
host.assert_provider_result();
assert_eq!(host.facts().cost, Cost::Deferred);
})
.await;
}
#[rstest]
#[case::complete(false)]
#[case::streaming(true)]
#[ignore = "calls a real provider, requires credentials and a live model"]
#[tokio::test]
async fn tool_round_trip(route: MessagesRoute, #[case] stream: bool) {
super::support::tool_round_trip(
route,
PROVIDER,
stream,
json!({
"max_tokens": 512, "tool_choice": {"type": "auto"}
}),
)
.await;
}

265
litellm-rust/crates/inference-messages/tests/live/support.rs Normal file
View file

@ -0,0 +1,265 @@
use std::{sync::OnceLock, time::Duration};
use futures_util::{FutureExt, TryStreamExt};
use litellm_host::interceptors::{
ExecutionFacts, Interceptors, RawResponse, RequestContext, ResultSource, WireRequest,
};
use litellm_inference_messages::{Error, MessagesCall, MessagesCallResponse, MessagesRoute};
use litellm_inference_testing::live::{LiveResources, model, within_deadline};
use litellm_llms_types::formats::messages::MessagesResponse;
use serde_json::{Value, json};
pub(super) struct LiveCall {
provider: &'static str,
facts: OnceLock<ExecutionFacts>,
}
impl LiveCall {
pub(super) fn new(provider: &'static str) -> Self {
Self {
provider,
facts: OnceLock::new(),
}
}
pub(super) fn facts(&self) -> &ExecutionFacts {
self.facts
.get()
.expect("the route must deliver execution facts")
}
pub(super) fn assert_provider_result(&self) {
let facts = self.facts();
assert_eq!(facts.provider.provider, self.provider);
assert_eq!(facts.source, ResultSource::Provider);
println!(
"{}",
json!({"route": "messages", "provider": self.provider, "source": "provider"})
);
}
}
impl Interceptors<Error> for LiveCall {
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
assert!(
self.facts.set(facts).is_ok(),
"result must be accepted exactly once"
);
Ok(())
}
async fn before_provider_request(
&self,
wire: WireRequest,
context: RequestContext,
) -> Result<WireRequest, Error> {
assert_eq!(context.custom_llm_provider, self.provider);
println!(
"{}",
json!({"route": "messages", "provider": self.provider, "model": context.model})
);
Ok(wire)
}
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
Ok(())
}
}
#[rstest::fixture]
pub(super) fn route() -> MessagesRoute {
let resources = LiveResources::default();
MessagesRoute::new(resources.http, resources.auth, resources.secrets)
}
pub(super) async fn request(
route: &MessagesRoute,
call: MessagesCall,
host: &LiveCall,
) -> MessagesCallResponse {
route
.execute(call, host, None)
.boxed()
.await
.expect("live request failed")
}
pub(super) fn call(provider: &str, payload: Value) -> MessagesCall {
let Value::Object(fields) = payload else {
panic!("live payload must be an object")
};
let body = serde_json::from_value(Value::Object(
[("max_tokens".into(), json!(128))]
.into_iter()
.chain(fields)
.chain([("model".into(), json!(model("messages", provider)))])
.collect(),
))
.expect("live payload must be a valid Messages request");
MessagesCall {
body,
api_key: None,
api_base: None,
custom_llm_provider: Some(provider.into()),
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: Some(Duration::from_secs(60)),
shaping: Default::default(),
}
}
pub(super) fn complete(response: MessagesCallResponse) -> Box<MessagesResponse> {
match response {
MessagesCallResponse::Complete(message) => message,
MessagesCallResponse::Stream { .. } => panic!("a complete request returned a stream"),
}
}
pub(super) fn assert_text(message: &MessagesResponse) {
assert!(!message.id.is_empty());
assert_eq!(message.role, "assistant");
assert!(message.stop_reason.is_some());
assert!(
message
.content
.iter()
.any(|block| block["text"].as_str().is_some_and(|text| !text.is_empty()))
);
assert!(
message
.usage
.as_ref()
.and_then(|usage| usage["output_tokens"].as_u64())
.is_some_and(|count| count > 0)
);
}
pub(super) async fn stream_events(response: MessagesCallResponse) -> Vec<Value> {
let MessagesCallResponse::Stream { head, chunks } = response else {
panic!("stream request returned a complete response")
};
let content_types: Vec<_> = head
.headers
.iter()
.filter(|(name, _)| name.eq_ignore_ascii_case("content-type"))
.map(|(_, value)| value.split(';').next().unwrap().trim())
.collect();
assert_eq!(content_types, ["text/event-stream"]);
let frames = litellm_framer::frames(chunks, litellm_framer::sse::SseCodec::default())
.try_collect::<Vec<_>>()
.await
.expect("live SSE framing failed");
let data_frames = match frames.last() {
Some(frame) if frame.data == "[DONE]" => &frames[..frames.len() - 1],
_ => frames.as_slice(),
};
let events: Vec<Value> = data_frames
.iter()
.map(|frame| serde_json::from_str(&frame.data).expect("SSE data must be JSON"))
.collect();
assert_eq!(
events.first().map(|event| event["type"].as_str()),
Some(Some("message_start"))
);
assert_eq!(
events.last().map(|event| event["type"].as_str()),
Some(Some("message_stop"))
);
assert!(!events.iter().any(|event| event["type"] == "error"));
assert!(events.iter().any(|event| {
event["usage"]["output_tokens"]
.as_u64()
.is_some_and(|count| count > 0)
}));
println!("{}", json!({"events": events}));
events
}
fn prompt() -> Value {
json!({"role": "user", "content": "Use the echo tool with value live-check, then tell me the result."})
}
fn tools() -> Value {
json!([{"name": "echo", "description": "Return a value", "input_schema": {
"type": "object", "properties": {"value": {"type": "string", "enum": ["live-check"]}}, "required": ["value"]
}}])
}
fn fragments(events: &[Value], index: u64, field: &str) -> String {
events
.iter()
.filter(|event| event["index"].as_u64() == Some(index))
.filter_map(|event| event["delta"][field].as_str())
.collect()
}
fn streamed_block(start: &Value, events: &[Value]) -> Value {
let index = start["index"]
.as_u64()
.expect("content block must have an index");
let block = &start["content_block"];
match block["type"].as_str() {
Some("tool_use") => {
let partial = fragments(events, index, "partial_json");
let input = match partial.is_empty() {
true => block["input"].clone(),
false => {
serde_json::from_str(&partial).expect("streamed tool arguments must be JSON")
}
};
json!({"type": "tool_use", "id": block["id"], "name": block["name"], "input": input})
}
Some("text") => {
json!({"type": "text", "text": format!("{}{}", block["text"].as_str().unwrap_or_default(), fragments(events, index, "text"))})
}
Some("thinking") => {
json!({"type": "thinking", "thinking": format!("{}{}", block["thinking"].as_str().unwrap_or_default(), fragments(events, index, "thinking")),
"signature": format!("{}{}", block["signature"].as_str().unwrap_or_default(), fragments(events, index, "signature"))})
}
_ => block.clone(),
}
}
pub(super) async fn tool_round_trip(
route: MessagesRoute,
provider: &'static str,
stream: bool,
tool_params: Value,
) {
within_deadline(async {
let first_host = LiveCall::new(provider);
let payload = json!({"messages": [prompt()], "tools": tools(), "stream": stream});
let fields = payload.as_object().expect("tool payload must be an object").clone();
let params = tool_params.as_object().expect("tool parameters must be an object").clone();
let first = request(&route, call(provider, Value::Object(fields.into_iter().chain(params).collect())), &first_host).await;
let content = match stream {
false => {
let message = complete(first);
assert_eq!(message.stop_reason.as_deref(), Some("tool_use"));
message.content
}
true => {
let events = stream_events(first).await;
assert!(events.iter().any(|event| event["delta"]["stop_reason"] == "tool_use"));
events.iter().filter(|event| event["type"] == "content_block_start")
.map(|event| streamed_block(event, &events)).collect()
}
};
let tool = content.iter().find(|block| block["type"] == "tool_use").expect("requested tool must be returned");
assert_eq!(tool["name"], "echo");
assert_eq!(tool["input"]["value"], "live-check");
let tool_id = tool["id"].as_str().filter(|id| !id.is_empty()).expect("tool must have an id");
first_host.assert_provider_result();
let second_host = LiveCall::new(provider);
let second = complete(request(&route, call(provider, json!({
"tools": tools(), "messages": [prompt(),
{"role": "assistant", "content": content},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": tool_id, "content": "live-check"}]}]
})), &second_host).await);
assert_text(&second);
second_host.assert_provider_result();
println!("{}", serde_json::to_string(&second).unwrap());
})
.await;
}

68
litellm-rust/crates/inference-messages/tests/live/vertex_ai.rs Normal file
View file

@ -0,0 +1,68 @@
use litellm_inference_messages::MessagesRoute;
use litellm_inference_testing::live::within_deadline;
use rstest::rstest;
use serde_json::{Value, json};
use super::support::{LiveCall, assert_text, call, complete, request, route, stream_events};
const PROVIDER: &str = "vertex_ai";
#[rstest]
#[case::basic(json!({"messages": [{"role": "user", "content": "Say hello."}]}))]
#[case::system_text(json!({
"system": "You are a helpful assistant.",
"messages": [{"role": "user", "content": "Say hello."}]
}))]
#[ignore = "calls a real provider, requires credentials and a live model"]
#[tokio::test]
async fn completion(route: MessagesRoute, #[case] payload: Value) {
within_deadline(async {
let host = LiveCall::new(PROVIDER);
let message = complete(request(&route, call(PROVIDER, payload), &host).await);
assert_text(&message);
host.assert_provider_result();
println!("{}", serde_json::to_string(&message).unwrap());
})
.await;
}
#[rstest]
#[ignore = "calls a real provider, requires credentials and a live model"]
#[tokio::test]
async fn streaming(route: MessagesRoute) {
within_deadline(async {
let host = LiveCall::new(PROVIDER);
let response = request(
&route,
call(
PROVIDER,
json!({"stream": true, "messages": [{"role": "user", "content": "Say hello."}]}),
),
&host,
)
.await;
let events = stream_events(response).await;
assert!(events.iter().any(|event| {
event["delta"]["text"]
.as_str()
.is_some_and(|text| !text.is_empty())
}));
host.assert_provider_result();
})
.await;
}
#[rstest]
#[case::complete(false)]
#[case::streaming(true)]
#[ignore = "calls a real provider, requires credentials and a live model"]
#[tokio::test]
async fn tool_round_trip(route: MessagesRoute, #[case] stream: bool) {
super::support::tool_round_trip(
route,
PROVIDER,
stream,
json!({"tool_choice": {"type": "auto"}}),
)
.await;
}

1
litellm-rust/crates/inference-messages/tests/messages/main.rs
View file

@ -24,6 +24,7 @@ mod request;
mod response;
mod secrets;
mod stream;
mod tencent;
const MODEL: &str = "claude-sonnet-4-5";

96
litellm-rust/crates/inference-messages/tests/messages/request.rs
View file

@ -11,6 +11,9 @@ use super::*;
#[rstest]
#[case::anthropic_key("anthropic", Some("sk-ant"), &[], ("x-api-key", "sk-ant"), &["authorization"])]
#[case::minimax_key("minimax", Some("sk-test"), &[], ("x-api-key", "sk-test"), &["authorization"])]
#[case::minimax_forwarded_key("minimax", Some("deployment"), &[("X-Api-Key", "caller")], ("x-api-key", "caller"), &["authorization"])]
#[case::minimax_forwarded_bearer("minimax", None, &[("Authorization", "Bearer caller")], ("authorization", "Bearer caller"), &["x-api-key"])]
#[case::azure_key("azure_ai", Some("sk-azure"), &[], ("x-api-key", "sk-azure"), &["authorization"])]
#[case::deepseek_key("deepseek", Some("sk-deepseek"), &[], ("x-api-key", "sk-deepseek"), &["authorization"])]
#[case::deepseek_forwards_caller_authorization(
@ -88,6 +91,7 @@ async fn credentials_become_exactly_one_auth_header(
#[case::anthropic("anthropic")]
#[case::azure_ai("azure_ai")]
#[case::deepseek("deepseek")]
#[case::minimax("minimax")]
#[tokio::test]
async fn a_call_without_credentials_fails_before_sending(
call: MessagesCall,
@ -123,6 +127,9 @@ async fn a_call_without_credentials_fails_before_sending(
"/v1/messages",
"/v1/messages"
)]
#[case::minimax(MODEL, Some("minimax"), "", "/v1/messages")]
#[case::minimax_prefix("minimax/claude-sonnet-4-5", None, "/v1/messages", "/v1/messages")]
#[case::minimax_endpoint_slash(MODEL, Some("minimax"), "/v1/messages/", "/v1/messages")]
#[case::azure_ai(MODEL, Some("azure_ai"), "", "/anthropic/v1/messages")]
#[case::deepseek(MODEL, Some("deepseek"), "", "/anthropic/v1/messages")]
#[case::deepseek_openai_compatible_base(MODEL, Some("deepseek"), "/beta", "/anthropic/v1/messages")]
@ -301,6 +308,7 @@ fn sent_betas(request: &wiremock::Request) -> BetaSet {
#[tokio::test]
async fn feature_betas_join_the_callers_betas_in_one_sorted_header(
call: MessagesCall,
#[values("anthropic", "minimax")] provider: &str,
#[case] fields: Value,
#[case] features: &[AnthropicBeta],
) {
@ -312,6 +320,7 @@ async fn feature_betas_join_the_callers_betas_in_one_sorted_header(
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
extra_headers: headers([("Anthropic-Beta", "caller-beta-2025-01-01")]),
@ -362,6 +371,7 @@ async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: M
#[case::anthropic("anthropic")]
#[case::azure_ai("azure_ai")]
#[case::deepseek("deepseek")]
#[case::minimax("minimax")]
#[tokio::test]
async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[case] provider: &str) {
let upstream = upstream([message_response()]).await;
@ -452,6 +462,7 @@ async fn unsupported_params_are_dropped_under_drop_params_and_rejected_without_i
#[tokio::test]
async fn reasoning_auto_summary_marks_active_thinking_on_the_wire(
call: MessagesCall,
#[values("anthropic", "minimax")] provider: &str,
#[case] thinking: Value,
#[case] expected: Value,
) {
@ -459,6 +470,7 @@ async fn reasoning_auto_summary_marks_active_thinking_on_the_wire(
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
shaping: MessagesShaping {
@ -517,6 +529,7 @@ async fn reasoning_auto_summary_marks_active_thinking_on_the_wire(
#[tokio::test]
async fn reasoning_is_translated_by_the_model_capabilities(
call: MessagesCall,
#[values("anthropic", "minimax")] provider: &str,
#[case] capabilities: MessagesModelCapabilities,
#[case] fields: Value,
#[case] expected: Value,
@ -525,6 +538,7 @@ async fn reasoning_is_translated_by_the_model_capabilities(
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
shaping: MessagesShaping {
@ -592,6 +606,7 @@ async fn replayed_history_is_cleaned_before_sending(
#[case::anthropic("anthropic")]
#[case::azure("azure_ai")]
#[case::deepseek("deepseek")]
#[case::minimax("minimax")]
#[tokio::test]
async fn metadata_is_reduced_to_the_user_id(call: MessagesCall, #[case] provider: &str) {
let upstream = upstream([message_response()]).await;
@ -703,11 +718,16 @@ async fn vertex_ai_addresses_the_model_in_the_url_and_not_in_the_body(
#[case::numeric_user_id(json!({"metadata": {"user_id": 7}}))]
#[case::missing_max_tokens(json!({"max_tokens": null}))]
#[tokio::test]
async fn an_invalid_request_fails_before_sending(call: MessagesCall, #[case] fields: Value) {
async fn an_invalid_request_fails_before_sending(
call: MessagesCall,
#[values("anthropic", "minimax")] provider: &str,
#[case] fields: Value,
) {
let upstream = upstream([message_response()]).await;
let error = run(with_fields(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
..call
@ -721,6 +741,45 @@ async fn an_invalid_request_fails_before_sending(call: MessagesCall, #[case] fie
assert!(received(&upstream).await.is_empty());
}
#[rstest]
#[case::billing_string(json!("x-anthropic-billing-header: private"), None)]
#[case::billing_blocks(json!([{ "type": "text", "text": "x-anthropic-billing-header: private" }]), None)]
#[case::mixed_blocks(
json!([{ "type": "text", "text": "x-anthropic-billing-header: private" }, { "type": "text", "text": "keep" }]),
Some(json!([{ "type": "text", "text": "keep" }]))
)]
#[case::ordinary_system(json!("keep"), Some(json!("keep")))]
#[tokio::test]
async fn minimax_strips_only_billing_system_blocks(
call: MessagesCall,
#[case] system: Value,
#[case] expected: Option<Value>,
) {
let upstream = upstream([message_response()]).await;
let fields = json!({
"system": system,
"tools": [{"name": "echo", "input_schema": {"type": "object"}}],
"messages": [{"role": "user", "content": [{
"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}
}]}]
});
run_message(with_fields(
MessagesCall {
custom_llm_provider: Some("minimax".into()),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
..call
},
fields.clone(),
))
.await;
let sent = only_request(&upstream).await.json();
assert_eq!(sent.get("system"), expected.as_ref());
assert_eq!(sent["tools"], fields["tools"]);
assert_eq!(sent["messages"], fields["messages"]);
}
#[rstest]
#[case::azure("azure_ai", &[], json!([
{"type": "text", "text": "top level"},
@ -835,3 +894,38 @@ async fn provider_validation_runs_before_caller_parameter_removal(
);
assert!(received(&upstream).await.is_empty());
}
#[rstest]
#[tokio::test]
async fn edenai_preserves_native_thinking_and_strips_billing_and_cache_extensions(
call: MessagesCall,
) {
let upstream = upstream([message_response()]).await;
let thinking = json!({"type": "enabled", "budget_tokens": 512, "native_field": true});
run_message(MessagesCall {
custom_llm_provider: Some("edenai".into()),
api_base: Some(format!("{}/v3", upstream.uri())),
api_key: Some("eden-key".into()),
extra_headers: headers([("Anthropic-Beta", "unregistered-native-beta")]),
..with_fields(call, json!({
"thinking": thinking, "temperature": 0.25,
"system": [{"type": "text", "text": "x-anthropic-billing-header: hidden"},
{"type": "text", "text": "keep", "cache_control": {"type": "ephemeral", "ttl": "1h", "scope": "global"}}],
}))
}).await;
let sent = only_request(&upstream).await;
assert_eq!(sent.url.path(), "/v3/v1/messages");
assert_eq!(sent.header("authorization"), Some("Bearer eden-key"));
assert_eq!(
sent.header("anthropic-beta"),
Some("unregistered-native-beta")
);
assert_eq!(
sent.json(),
json!({
"model": MODEL, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}],
"thinking": thinking, "temperature": 0.25,
"system": [{"type": "text", "text": "keep", "cache_control": {"type": "ephemeral"}}],
})
);
}

7
litellm-rust/crates/inference-messages/tests/messages/secrets.rs
View file

@ -25,6 +25,13 @@ use super::*;
"/anthropic/v1/messages",
&["DEEPSEEK_API_KEY", "DEEPSEEK_ANTHROPIC_API_BASE", "DEEPSEEK_API_BASE"]
)]
#[case::minimax(
"minimax",
"MINIMAX_API_KEY",
"MINIMAX_API_BASE",
"/v1/messages",
&["MINIMAX_API_KEY", "MINIMAX_API_BASE"]
)]
#[tokio::test]
async fn the_credential_and_base_come_from_the_secret_source(
call: MessagesCall,

18
litellm-rust/crates/inference-messages/tests/messages/stream.rs
View file

@ -150,9 +150,21 @@ async fn stream_through(host: &RecordingStreamHost) -> Result<MessagesOutput, Er
#[rstest]
#[tokio::test]
async fn upstream_headers_are_on_the_stream_head_before_the_first_chunk(call: MessagesCall) {
async fn upstream_headers_are_on_the_stream_head_before_the_first_chunk(
call: MessagesCall,
#[values("anthropic", "minimax")] provider: &str,
) {
let upstream = upstream([sse_response()]).await;
let host = RecordingStreamHost::new(streaming(call, upstream.uri()), usize::MAX);
let host = RecordingStreamHost::new(
streaming(
MessagesCall {
custom_llm_provider: Some(provider.into()),
..call
},
upstream.uri(),
),
usize::MAX,
);
let outcome = stream_through(&host).await.expect("streamed call succeeds");
@ -336,6 +348,7 @@ async fn the_timeout_covers_a_stalled_stream_body(call: MessagesCall) {
#[rstest]
#[case::anthropic("anthropic")]
#[case::azure_ai("azure_ai")]
#[case::edenai("edenai")]
#[tokio::test]
async fn the_sdk_returns_stream_headers_and_every_sse_byte(
call: MessagesCall,
@ -346,6 +359,7 @@ async fn the_sdk_returns_stream_headers_and_every_sse_byte(
.execute(
MessagesCall {
custom_llm_provider: Some(provider.into()),
api_key: Some("stream-key".into()),
litellm_params: Default::default(),
..streaming(call, upstream.uri())
},

260
litellm-rust/crates/inference-messages/tests/messages/tencent.rs Normal file
View file

@ -0,0 +1,260 @@
use std::ops::ControlFlow;
use bytes::Bytes;
use litellm_llms::base_llm::messages::context::MessagesModelCapabilities;
use litellm_llms_types::providers::anthropic::DEFAULT_HEADERS;
use rstest::{fixture, rstest};
use super::*;
#[fixture]
fn tencent_call(call: MessagesCall) -> MessagesCall {
MessagesCall {
custom_llm_provider: Some("tencent".into()),
api_key: Some("deployment-key".into()),
..call
}
}
#[rstest]
#[case::deployment_key(&[], "x-api-key", "deployment-key")]
#[case::forwarded_key(&[("X-Api-Key", "caller-key")], "x-api-key", "caller-key")]
#[case::forwarded_bearer(&[("Authorization", "Bearer caller")], "authorization", "Bearer caller")]
#[tokio::test]
async fn credentials_reach_the_selected_provider(
tencent_call: MessagesCall,
#[case] forwarded: &[(&str, &str)],
#[case] header: &str,
#[case] credential: &str,
) {
let upstream = upstream([message_response()]).await;
run_message(MessagesCall {
api_base: Some(upstream.uri()),
extra_headers: headers(forwarded.iter().copied()),
..tencent_call
})
.await;
let sent = only_request(&upstream).await;
assert_eq!(sent.header_values(header), [credential]);
let version = DEFAULT_HEADERS
.iter()
.find_map(|(name, value)| (*name == "anthropic-version").then_some(*value))
.unwrap();
assert_eq!(sent.header_values("anthropic-version"), [version]);
assert_eq!(sent.json()["model"], MODEL);
}
#[rstest]
#[tokio::test]
async fn credentials_and_base_resolve_from_the_secret_source(tencent_call: MessagesCall) {
let upstream = upstream([message_response()]).await;
let base = format!("{}/v1/chat/completions/", upstream.uri());
let secrets = Arc::new(RecordingSecrets::new([
("TENCENT_API_KEY", "secret-key"),
("TENCENT_ANTHROPIC_API_BASE", base.as_str()),
("TENCENT_API_BASE", UNREACHABLE_BASE),
]));
run_with(
secrets,
MessagesCall {
api_key: None,
..tencent_call
},
)
.await
.unwrap();
let sent = only_request(&upstream).await;
assert_eq!(sent.url.path(), "/v1/messages");
assert_eq!(sent.header("x-api-key"), Some("secret-key"));
}
#[rstest]
#[tokio::test]
async fn missing_credentials_fail_before_sending(tencent_call: MessagesCall) {
let upstream = upstream([message_response()]).await;
let error = run(MessagesCall {
api_key: None,
api_base: Some(upstream.uri()),
..tencent_call
})
.await
.unwrap_err();
assert!(matches!(
error,
Error::Auth(litellm_auth::Error::MissingApiKey {
environment_variable: "TENCENT_API_KEY",
..
})
));
assert!(received(&upstream).await.is_empty());
}
#[rstest]
#[case::missing_max_tokens(json!({"max_tokens": null}))]
#[case::invalid_metadata(json!({"metadata": {"user_id": 7}}))]
#[case::invalid_reasoning_effort(json!({"reasoning_effort": "unknown"}))]
#[tokio::test]
async fn malformed_requests_fail_before_sending(tencent_call: MessagesCall, #[case] fields: Value) {
let upstream = upstream([message_response()]).await;
let error = run(with_fields(
MessagesCall {
api_base: Some(upstream.uri()),
..tencent_call
},
fields,
))
.await
.unwrap_err();
assert!(error.is_request(), "{error:?}");
assert!(received(&upstream).await.is_empty());
}
#[rstest]
#[case::legacy(false, json!({"type": "enabled", "budget_tokens": 4095}))]
#[case::adaptive(true, json!({"type": "adaptive", "display": "summarized"}))]
#[tokio::test]
async fn reasoning_effort_becomes_model_appropriate_thinking(
tencent_call: MessagesCall,
#[case] adaptive: bool,
#[case] thinking: Value,
) {
let upstream = upstream([message_response()]).await;
run_message(with_fields(
MessagesCall {
api_base: Some(upstream.uri()),
shaping: MessagesShaping {
capabilities: MessagesModelCapabilities {
supports_reasoning: true,
supports_adaptive_thinking: adaptive,
..Default::default()
},
..Default::default()
},
..tencent_call
},
json!({"max_tokens": 4096, "reasoning_effort": "high"}),
))
.await;
let sent = only_request(&upstream).await.json();
assert_eq!(sent["thinking"], thinking);
assert_eq!(sent.get("reasoning_effort"), None);
assert_eq!(
sent.get("output_config"),
adaptive.then(|| json!({"effort": "high"})).as_ref()
);
}
#[rstest]
#[case::encrypted_reasoning(json!({"type": "thinking", "thinking": "private", "signature": "litellm_encrypted_reasoning:private"}))]
#[case::orphaned_advisor(json!({"type": "server_tool_use", "name": "advisor", "id": "srvtoolu_advisor", "input": {}}))]
#[tokio::test]
async fn replayed_history_is_cleaned_without_losing_text(
tencent_call: MessagesCall,
#[case] block: Value,
) {
let upstream = upstream([message_response()]).await;
run_message(with_fields(
MessagesCall {
api_base: Some(upstream.uri()),
..tencent_call
},
json!({"messages": [
{"role": "assistant", "content": [block, {"type": "text", "text": "keep"}]},
{"role": "user", "content": "continue"}
]}),
))
.await;
assert_eq!(
only_request(&upstream).await.json()["messages"],
json!([
{"role": "assistant", "content": [{"type": "text", "text": "keep"}]},
{"role": "user", "content": "continue"}
])
);
}
#[rstest]
#[case::billing_string(json!("x-anthropic-billing-header: private"), None)]
#[case::mixed_blocks(json!([
{"type": "text", "text": "x-anthropic-billing-header: private"},
{"type": "text", "text": "keep"}
]), Some(json!([{ "type": "text", "text": "keep" }])))]
#[case::ordinary_system(json!("keep"), Some(json!("keep")))]
#[tokio::test]
async fn billing_is_removed_and_tools_and_cache_hints_are_preserved(
tencent_call: MessagesCall,
#[case] system: Value,
#[case] expected: Option<Value>,
) {
let upstream = upstream([message_response()]).await;
let fields = json!({
"system": system,
"tools": [{"name": "echo", "input_schema": {"type": "object"}}],
"messages": [{"role": "user", "content": [{
"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}
}]}]
});
run_message(with_fields(
MessagesCall {
api_base: Some(upstream.uri()),
..tencent_call
},
fields.clone(),
))
.await;
let sent = only_request(&upstream).await.json();
assert_eq!(sent.get("system"), expected.as_ref());
assert_eq!(sent["tools"], fields["tools"]);
assert_eq!(sent["messages"], fields["messages"]);
}
#[derive(Default)]
struct StreamSink(Mutex<Vec<u8>>);
impl litellm_host_native::in_process::StreamConsumer<Messages> for StreamSink {
async fn open_stream(
&self,
_: litellm_inference_messages::route::MessagesStreamHead,
) -> Result<ControlFlow<()>, Error> {
Ok(ControlFlow::Continue(()))
}
async fn send_chunk(&self, chunk: Bytes) -> Result<ControlFlow<()>, Error> {
self.0.lock().unwrap().extend_from_slice(&chunk);
Ok(ControlFlow::Continue(()))
}
}
#[rstest]
#[tokio::test]
async fn streaming_relays_provider_events_unchanged(tencent_call: MessagesCall) {
let events = concat!(
"event: message_start\ndata: {\"type\":\"message_start\"}\n\n",
"event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"OK\"}}\n\n",
"event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"
);
let upstream =
upstream([ResponseTemplate::new(200).set_body_raw(events, "text/event-stream")]).await;
let host = LocalMessagesHost::new(with_fields(
MessagesCall {
api_base: Some(upstream.uri()),
..tencent_call
},
json!({"stream": true}),
));
let sink = StreamSink::default();
let outcome = litellm_host_native::in_process::run_hosted(
machine(Arc::new(RecordingSecrets::empty()))(host.request().unwrap()),
litellm_host_native::in_process::Host {
services: &(),
interceptors: &host,
stream: &sink,
observers: None,
},
)
.await
.unwrap();
assert!(matches!(outcome, MessagesOutput::StreamEnded));
assert_eq!(sink.0.into_inner().unwrap(), events.as_bytes());
assert_eq!(only_request(&upstream).await.json()["stream"], true);
}

147
litellm-rust/crates/inference-messages/tests/openrouter/main.rs Normal file
View file

@ -0,0 +1,147 @@
use std::{sync::Arc, time::Duration};
use futures_util::TryStreamExt;
use litellm_inference_messages::{Error, MessagesCall, MessagesCallResponse, MessagesShaping};
use litellm_inference_testing::{RecordingSecrets, no_secrets};
use rstest::{fixture, rstest};
use serde_json::json;
use wiremock::ResponseTemplate;
#[path = "../support/mod.rs"]
mod support;
use support::*;
#[fixture]
fn call() -> MessagesCall {
MessagesCall {
body: serde_json::from_value(json!({"model": "openrouter/anthropic/test-model", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})).unwrap(),
api_key: None,
api_base: None,
custom_llm_provider: None,
litellm_params: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: Some(Duration::from_secs(5)),
shaping: MessagesShaping::default(),
}
}
fn message_response() -> ResponseTemplate {
json_response(
json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "anthropic/test-model", "content": [{"type": "text", "text": "hi"}], "stop_reason": "end_turn", "usage": {"input_tokens": 1, "output_tokens": 1}}),
)
}
#[rstest]
#[case::explicit(Some("explicit"), None, "Bearer explicit")]
#[case::managed(None, None, "Bearer managed")]
#[case::forwarded(None, Some("bEaReR caller"), "bEaReR caller")]
#[case::blank(Some("explicit"), Some("Bearer "), "Bearer explicit")]
#[case::malformed(Some("explicit"), Some("invali\u{00e9}auth"), "Bearer explicit")]
#[tokio::test]
async fn only_a_valid_bearer_can_override_the_configured_key(
call: MessagesCall,
#[case] api_key: Option<&str>,
#[case] authorization: Option<&str>,
#[case] expected: &str,
) {
let upstream = upstream([message_response()]).await;
let extra_headers = [("x-api-key".to_string(), json!("caller-anthropic-key"))]
.into_iter()
.chain(authorization.map(|value| ("Authorization".to_string(), json!(value))))
.collect();
let secrets = Arc::new(RecordingSecrets::new([("OPENROUTER_API_KEY", "managed")]));
let output = messages_route(secrets)
.execute(
MessagesCall {
api_key: api_key.map(str::to_string),
api_base: Some(format!("{}/api/v1", upstream.uri())),
extra_headers: Some(extra_headers),
..call
},
&(),
None,
)
.await
.unwrap();
assert!(matches!(output, MessagesCallResponse::Complete(_)));
let request = only_request(&upstream).await;
assert_eq!(request.url.path(), "/api/v1/messages");
assert_eq!(request.header_values("authorization"), [expected]);
assert_eq!(request.header("x-api-key"), None);
assert_eq!(request.json()["model"], "anthropic/test-model");
}
#[rstest]
#[tokio::test]
async fn an_anthropic_key_alone_is_not_an_openrouter_credential(call: MessagesCall) {
let upstream = upstream([message_response()]).await;
let result = messages_route(no_secrets())
.execute(
MessagesCall {
api_base: Some(upstream.uri()),
extra_headers: Some(
json!({"x-api-key": "caller-key"})
.as_object()
.unwrap()
.clone(),
),
..call
},
&(),
None,
)
.await;
assert!(matches!(
result,
Err(Error::Auth(litellm_auth::Error::MissingApiKey {
provider: "OpenRouter",
..
}))
));
assert!(received(&upstream).await.is_empty());
}
#[rstest]
#[tokio::test]
async fn the_secret_manager_supplies_the_native_base_and_key(call: MessagesCall) {
let upstream = upstream([message_response()]).await;
let base = format!("{}/api/v1", upstream.uri());
let secrets = Arc::new(RecordingSecrets::new([
("OPENROUTER_API_KEY", "managed"),
("OPENROUTER_API_BASE", base.as_str()),
]));
messages_route(secrets)
.execute(call, &(), None)
.await
.unwrap();
let request = only_request(&upstream).await;
assert_eq!(request.url.path(), "/api/v1/messages");
assert_eq!(request.header("authorization"), Some("Bearer managed"));
}
#[rstest]
#[tokio::test]
async fn streaming_relay_preserves_every_event_and_billed_usage(call: MessagesCall) {
let sse = "event: message_delta\ndata: {\"type\":\"message_delta\",\"usage\":{\"cost\":\"0.001\",\"cost_details\":\"unexpected\"}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n";
let upstream =
upstream([ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")]).await;
let streamed = MessagesCall {
body: serde_json::from_value(json!({"model": "openrouter/anthropic/test-model", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}], "stream": true})).unwrap(),
api_key: Some("key".into()),
api_base: Some(upstream.uri()),
..call
};
let output = messages_route(no_secrets())
.execute(streamed, &(), None)
.await
.unwrap();
let MessagesCallResponse::Stream { chunks, .. } = output else {
panic!("expected SSE");
};
assert_eq!(
chunks.try_collect::<Vec<_>>().await.unwrap().concat(),
sse.as_bytes()
);
assert_eq!(only_request(&upstream).await.json()["stream"], true);
}

2
litellm-rust/crates/inference-testing/Cargo.toml
View file

@ -7,7 +7,9 @@ repository.workspace = true
publish = false
[dependencies]
litellm-auth.workspace = true
futures-util.workspace = true
litellm-http.workspace = true
litellm-inference.workspace = true
litellm-secrets.workspace = true
tokio.workspace = true

46
litellm-rust/crates/inference-testing/README.md Normal file
View file

@ -0,0 +1,46 @@
# Live provider route tests
Each route crate owns `tests/live.rs`. Use named `rstest` provider cases for the behaviors that route supports. Shared setup lives in `litellm_inference_testing::live`, with production HTTP settings, authentication services, environment secrets and a deadline
These tests execute the Rust route against a real provider. They do not prove Python parity, proxy authorization, billing persistence or agent compatibility. Those belong in the corresponding boundary and gateway suites
All live cases use `#[ignore = "calls a real provider, requires credentials and a live model"]`. Normal Cargo runs report them as ignored. An explicitly selected case fails on missing credentials, unavailable models, provider errors or timeouts. Never return early and count missing coverage as a pass
Set `LITELLM_LIVE_<ROUTE>_<PROVIDER>_MODEL` to the latest model available to the test account. The model has no compiled default because vendor availability changes independently of LiteLLM. Credentials use the provider's existing configuration rather than a second test-only key convention. Export credentials in the shell before running tests. The harness does not load `.env` or modify process environment while tests run
## Messages
Completion checks text and usage. Streaming consumes SSE to completion and checks text, usage and the terminal event. Tool round trips request a tool call and send its returned ID back with a result. Each case checks provider execution facts and rejects cache results
Messages setup and assertions live in `inference-messages/tests/live/support.rs`. Provider modules under `tests/live/` select their payloads and tool policy. The shared helpers consume streams completely and reconstruct text, thinking signatures and tool arguments for the follow-up request
The initial provider cases are `vertex_ai` and `edenai`. A registered ignored case is available coverage, not evidence that it passed against that provider
Run one provider's cases explicitly
```bash
LITELLM_LIVE_MESSAGES_VERTEX_AI_MODEL=<claude-model-on-vertex> \
cargo test --manifest-path litellm-rust/Cargo.toml \
-p litellm-inference-messages --test live vertex_ai \
-- --ignored --nocapture --test-threads=1
```
For Vertex AI, set `LITELLM_LIVE_MESSAGES_VERTEX_AI_MODEL` to a Claude model published in the project's location. Credentials, project and location resolve through the production Rust GCP authentication, from `GOOGLE_APPLICATION_CREDENTIALS` or `VERTEXAI_CREDENTIALS`, with `VERTEXAI_PROJECT` and `VERTEXAI_LOCATION`
Vertex AI cases live in `inference-messages/tests/live/vertex_ai.rs`. They cover plain and system messages, text streaming, and complete and streaming tool round trips. Tool cases use automatic selection because current Claude models reject forced `tool_choice`. Each case still requires a tool call, checks its arguments, and sends the tool result back
For EdenAI, set `LITELLM_LIVE_MESSAGES_EDENAI_MODEL` and `EDENAI_API_KEY`, and select `edenai`. EdenAI requires a production token for live coverage, because sandbox tokens return dummy responses
EdenAI cases live in `inference-messages/tests/live/edenai.rs`. They cover system messages, portable cache hints, provider-reported cost handoff, text streaming, and complete and streaming tool round trips. The cost assertion compares the public response with the execution facts instead of pinning a vendor price
EdenAI tool cases use automatic selection because current Claude models reject forced `tool_choice`. The test still requires a tool call and verifies its arguments and follow-up response
Other tool cases require a model that supports forced `tool_choice`, except where a provider's section says otherwise. A provider rejection fails the selected case instead of skipping it
Select one behavior by its test name, for example `completion::case_1_basic`. To list every available cell without contacting any provider
```bash
cargo test --manifest-path litellm-rust/Cargo.toml \
-p litellm-inference-messages --test live -- --list
```
For another route, reuse `LiveResources`, `model` and `within_deadline`. Keep payload construction and response assertions in that route's test file. Add only supported provider behaviors, and document any gap instead of substituting mocks or skipping failures. Run cases serially to keep spending and rate limits predictable

2
litellm-rust/crates/inference-testing/src/lib.rs
View file

@ -9,6 +9,8 @@ use litellm_secrets::{SecretValue, source::SecretSource};
use litellm_inference::resources::CoreResources;
pub mod live;
pub fn http_pool() -> HttpClientPool {
HttpClientPool::new(Arc::new(PublicDnsResolver))
}

53
litellm-rust/crates/inference-testing/src/live.rs Normal file
View file

@ -0,0 +1,53 @@
use std::{future::Future, sync::Arc, time::Duration};
use litellm_http::{ClientVariant, HttpSettings, HttpSettingsLayer, Resolution};
use litellm_inference::resources::CoreResources;
use litellm_secrets::source::{EnvironmentSecrets, SecretSource};
pub const DEADLINE: Duration = Duration::from_secs(120);
pub struct LiveResources {
pub http: litellm_http::Client,
pub auth: Arc<litellm_auth::AuthServices>,
pub secrets: Arc<dyn SecretSource>,
}
impl Default for LiveResources {
fn default() -> Self {
let resources = CoreResources::new(Arc::new(super::http_pool()));
let settings =
HttpSettings::from_layers([HttpSettingsLayer::from_environment(&|key: &str| {
std::env::var(key).ok()
})]);
let resolution = Resolution::from(&settings);
let http = resources
.pool
.client(&resolution.config, ClientVariant::Provider)
.expect("live provider HTTP settings must be valid");
let secrets_http = resources
.pool
.client(&resolution.config, ClientVariant::NoRedirect)
.expect("live secret HTTP settings must be valid");
Self {
http,
auth: resources.auth,
secrets: Arc::new(EnvironmentSecrets::python_compatible(secrets_http)),
}
}
}
pub fn model(route: &str, provider: &str) -> String {
let name = format!("LITELLM_LIVE_{route}_{provider}_MODEL").to_ascii_uppercase();
std::env::var(&name)
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| {
panic!("set {name} to an available current model before running this live case")
})
}
pub async fn within_deadline<T>(operation: impl Future<Output = T>) -> T {
tokio::time::timeout(DEADLINE, operation)
.await
.expect("live case exceeded its 120 second deadline")
}

7
litellm-rust/crates/inference/src/caching.rs
View file

@ -13,7 +13,9 @@ use litellm_cache_response::{
};
use litellm_host::{
call::{CallOutput, OutputOf},
interceptors::{ExecutionFacts, Interceptors, ProviderIdentity, ResultSource, WireRequest},
interceptors::{
Cost, ExecutionFacts, Interceptors, ProviderIdentity, ResultSource, WireRequest,
},
lifecycle::{CallEvent, ExecutionEvent},
observation::ObservationSender,
protocol::Protocol,
@ -168,10 +170,12 @@ where
None => (provider().await?, ResultSource::Provider),
};
let from_provider = source == ResultSource::Provider;
let cost = Cost::settle(&source, None);
publish(
ExecutionFacts {
provider: identity,
source,
cost,
},
interceptors,
observers,
@ -212,6 +216,7 @@ where
publish(
ExecutionFacts {
provider: identity,
cost: Cost::settle(&source, None),
source: source.clone(),
},
interceptors,

2
litellm-rust/crates/llms/AGENTS.md
View file

@ -24,7 +24,7 @@
- Every file is `# rules` then `# references`, both concise bullets. Split a long `# rules` into `##` topic sections
- A folder gets an AGENTS.md only when it adds something: a deviation, an explicit reuse of another provider, or upstream docs no other file lists. A folder without one follows its nearest parent
- Every `src/<provider>/<format>/` folder gets one, since it talks to its own upstream endpoint. `.greptile/files.json` lists them all
- Every `src/<provider>/<format>/` folder gets one, since it talks to its own upstream endpoint
- A rule lives in the highest file where it holds. Never restate a parent's rule
- Each upstream URL appears in exactly one AGENTS.md, the one owning that contract. Others point to that file by path
- Format specs: `litellm-llms-types/src/formats/<format>/`

4
litellm-rust/crates/llms/src/base_llm/messages/transformation.rs
View file

@ -41,6 +41,10 @@ pub trait BaseMessagesConfig: Sync {
Ok(response)
}
fn reported_cost(&self, _response: &MessagesResponse) -> Option<serde_json::Number> {
None
}
fn secret_names(&self) -> &'static [&'static str];
/// Shapes the forwarded headers and names the credential, the way Python's

11
litellm-rust/crates/llms/src/bedrock/claude_platform/messages/AGENTS.md Normal file
View file

@ -0,0 +1,11 @@
# rules
- Reuse Anthropic Messages shaping, reasoning translation, and beta headers
- Require a workspace header and use an API key or AWS SigV4
- Keep connection fields out of the upstream request body
- Drop `context_management` by default, with per-request `claude_platform_unsupported_params` overrides
# references
- Python counterpart `litellm/llms/bedrock/claude_platform/messages_transformation.py`
- https://platform.claude.com/docs/en/build-with-claude/claude-platform-on-aws

1
litellm-rust/crates/llms/src/bedrock/claude_platform/messages/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod transformation;

395
litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs Normal file
View file

@ -0,0 +1,395 @@
use std::sync::LazyLock;
use litellm_auth::{AwsParams, CredentialPlacement, ParamSpec, SecretValue};
use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region};
use litellm_core_utils::settings::resolve_non_empty;
use litellm_http::request::{header_value, with_header, without_headers};
use litellm_llms_types::{
formats::messages::MessagesRequest,
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
use serde_json::Value;
use crate::{
Error, ErrorDetail,
anthropic::messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
},
base_llm::{
auth::{AuthScheme, Headers, ValidatedEnvironment},
messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig},
},
bedrock::common_utils::validated_region,
};
const SERVICE: &str = "aws-external-anthropic";
const BASE_ENVS: &[&str] = &["ANTHROPIC_AWS_BASE_URL", "ANTHROPIC_AWS_API_BASE"];
const WORKSPACE_ENVS: &[&str] = &["ANTHROPIC_AWS_WORKSPACE_ID", "ANTHROPIC_WORKSPACE_ID"];
const WORKSPACE_FIELDS: &[&str] = &[
"workspace_id",
"aws_workspace_id",
"anthropic-workspace-id",
"anthropic_workspace_id",
];
const WORKSPACE_HEADER: &str = "anthropic-workspace-id";
static WORKSPACE: ParamSpec = ParamSpec {
setting: "workspace_id",
wire: WORKSPACE_FIELDS,
module_global: None,
env: WORKSPACE_ENVS,
};
const UNSUPPORTED_OVERRIDE: &str = "claude_platform_unsupported_params";
pub struct BedrockClaudePlatformMessagesConfig;
pub const BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG: BedrockClaudePlatformMessagesConfig =
BedrockClaudePlatformMessagesConfig;
impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
fn shape_request(
&self,
request: MessagesRequest,
reasoning_auto_summary: bool,
) -> Result<MessagesRequest, Error> {
shape_anthropic_messages_request(request, reasoning_auto_summary)
}
fn validate_environment(
&self,
headers: Headers,
api_key: Option<&str>,
_model: &str,
params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
let workspace = header_value(&headers, WORKSPACE_HEADER)
.filter(|value| !value.trim().is_empty())
.or_else(|| {
WORKSPACE_FIELDS.iter().find_map(|field| {
params
.extra
.get(*field)?
.as_str()
.filter(|value| !value.trim().is_empty())
})
});
let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS)
.ok_or_else(|| Error::Auth(WORKSPACE.missing("Claude Platform on AWS")))?;
let headers = with_header(headers, WORKSPACE_HEADER, workspace);
let key = resolve_non_empty(api_key, env, &["ANTHROPIC_AWS_API_KEY"]);
let headers = without_headers(headers, &["authorization"]);
if header_value(&headers, "x-api-key").is_some_and(|value| !value.trim().is_empty()) {
return Ok(ValidatedEnvironment {
headers,
auth: AuthScheme::Forwarded,
});
}
if let Some(key) = key {
return Ok(ValidatedEnvironment {
headers: without_headers(headers, &["x-api-key"]),
auth: AuthScheme::Credential {
placement: CredentialPlacement::Header("x-api-key"),
secret: SecretValue::new(key),
},
});
}
let base = resolve_non_empty(params.api_base.as_deref(), env, BASE_ENVS);
Ok(ValidatedEnvironment {
headers: without_headers(headers, &["x-api-key"]),
auth: AuthScheme::AwsSigV4 {
region: platform_region(base.as_deref(), params, env)?,
service: SERVICE,
credentials: Box::new(AwsCredentialSource::from_params(&params.aws, env)),
},
})
}
fn get_complete_url(
&self,
api_base: Option<&str>,
_model: &str,
params: &LitellmParams,
_stream: bool,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let base = resolve_non_empty(
api_base
.filter(|base| !base.trim().is_empty())
.or(params.api_base.as_deref()),
env,
BASE_ENVS,
);
let base = match base {
Some(base) => base,
None => format!(
"https://{SERVICE}.{}.api.aws",
platform_region(None, params, env)?
),
};
let base = base.trim_end_matches('/');
Ok(if base.ends_with(MESSAGES_PATH) {
base.into()
} else {
format!("{base}{MESSAGES_PATH}")
})
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
let body = serde_json::to_value(request).map_err(invalid_request)?;
let filtered = serde_json::from_value(filter_body(body, false)).map_err(invalid_request)?;
transform_messages_request(filtered, context)
}
fn secret_names(&self) -> &'static [&'static str] {
static NAMES: LazyLock<Box<[&'static str]>> = LazyLock::new(|| {
["ANTHROPIC_AWS_API_KEY"]
.into_iter()
.chain(BASE_ENVS.iter().copied())
.chain(WORKSPACE_ENVS.iter().copied())
.chain(["AWS_DEFAULT_REGION"])
.chain(AwsParams::secret_names().iter().copied())
.collect()
});
&NAMES
}
fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
DEFAULT_HEADERS
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
let headers = match WORKSPACE_FIELDS.iter().find_map(|field| {
request
.params
.extra
.get(*field)?
.as_str()
.filter(|value| !value.trim().is_empty())
}) {
Some(workspace) => with_header(headers, WORKSPACE_HEADER, workspace.into()),
None => headers,
};
update_headers_with_anthropic_beta(headers, request)
}
fn wire_body(&self, body: Value) -> Value {
filter_body(body, true)
}
}
fn platform_region(
base: Option<&str>,
params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let host_region = base.and_then(|base| {
let url = url::Url::parse(base).ok()?;
url.host_str()?
.strip_prefix("aws-external-anthropic.")?
.strip_suffix(".api.aws")
.map(str::to_string)
});
let region = host_region
.or_else(|| resolve_aws_region(None, &params.aws, env))
.or_else(|| resolve_non_empty(None, env, &["AWS_DEFAULT_REGION"]))
.ok_or_else(|| Error::Auth(AwsParams::REGION.missing("Claude Platform on AWS")))?;
validated_region(region)
}
fn filter_body(body: Value, strip_connection_fields: bool) -> Value {
let Value::Object(fields) = body else {
return body;
};
let unsupported: Vec<&str> = fields
.get(UNSUPPORTED_OVERRIDE)
.and_then(Value::as_array)
.map(|values| values.iter().filter_map(Value::as_str).collect())
.unwrap_or_else(|| vec!["context_management"]);
let retained = fields
.iter()
.filter(|(name, _)| {
!unsupported.contains(&name.as_str())
&& !(strip_connection_fields
&& (WORKSPACE_FIELDS.contains(&name.as_str())
|| name.starts_with("aws_")
|| name.as_str() == UNSUPPORTED_OVERRIDE))
})
.map(|(name, value)| {
let value = if name == "model" {
value
.as_str()
.map(|model| {
Value::String(
model
.strip_prefix("claude_platform/")
.unwrap_or(model)
.into(),
)
})
.unwrap_or_else(|| value.clone())
} else {
value.clone()
};
(name.clone(), value)
})
.collect();
Value::Object(retained)
}
fn invalid_request(error: serde_json::Error) -> Error {
Error::InvalidRequest(ErrorDetail::invalid("Messages request", error))
}
#[cfg(test)]
mod tests {
use super::*;
use litellm_llms_types::providers::anthropic::AnthropicBeta;
use rstest::rstest;
use serde_json::json;
#[rstest]
fn workspace_is_required_before_any_upstream_request() {
assert_eq!(
BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
.validate_environment(
vec![],
Some("key"),
"claude-test",
&LitellmParams::default(),
&|_| None
)
.unwrap_err(),
Error::Auth(WORKSPACE.missing("Claude Platform on AWS"))
);
}
#[rstest]
#[case::valid("forwarded", true)]
#[case::blank(" ", false)]
fn forwarded_keys_win_and_blank_keys_fall_back(#[case] key: &str, #[case] forwarded: bool) {
let environment = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
.validate_environment(
vec![
("X-Api-Key".into(), key.into()),
("authorization".into(), "stale".into()),
],
Some("explicit"),
"claude-test",
&LitellmParams::default(),
&|name| (name == "ANTHROPIC_AWS_WORKSPACE_ID").then(|| "workspace".into()),
)
.unwrap();
assert_eq!(
header_value(&environment.headers, WORKSPACE_HEADER),
Some("workspace")
);
assert_eq!(header_value(&environment.headers, "authorization"), None);
if forwarded {
assert_eq!(header_value(&environment.headers, "x-api-key"), Some(key));
assert!(matches!(environment.auth, AuthScheme::Forwarded));
} else {
assert_eq!(header_value(&environment.headers, "x-api-key"), None);
assert!(
matches!(environment.auth, AuthScheme::Credential { secret, placement: CredentialPlacement::Header("x-api-key") } if secret.expose() == "explicit")
);
}
}
#[rstest]
#[case::region("us-west-2", true)]
#[case::injection("us-east-1.attacker.test/steal?", false)]
fn generated_endpoint_regions_are_validated(#[case] region: &str, #[case] valid: bool) {
let result = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG.get_complete_url(
None,
"claude-test",
&LitellmParams::default(),
false,
&|name| (name == "AWS_DEFAULT_REGION").then(|| region.into()),
);
if valid {
assert_eq!(
result.unwrap(),
format!("https://{SERVICE}.{region}.api.aws/v1/messages")
);
} else {
assert!(matches!(result, Err(Error::InvalidRequest(_))));
}
}
#[rstest]
#[case::explicit_regional_endpoint(
"https://aws-external-anthropic.us-west-2.api.aws",
"us-east-1",
"us-west-2"
)]
fn public_regional_host_precedes_configured_region(
#[case] api_base: &str,
#[case] configured_region: &str,
#[case] expected_region: &str,
) {
let params = LitellmParams {
api_base: Some(api_base.into()),
aws: AwsParams {
aws_region_name: Some(configured_region.into()),
..Default::default()
},
..Default::default()
};
let env = |name: &str| (name == "ANTHROPIC_AWS_WORKSPACE_ID").then(|| "workspace".into());
let url = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
.get_complete_url(None, "claude-test", &params, false, &env)
.unwrap();
let environment = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
.validate_environment(vec![], None, "claude-test", &params, &env)
.unwrap();
assert_eq!(
url,
"https://aws-external-anthropic.us-west-2.api.aws/v1/messages"
);
assert!(
matches!(environment.auth, AuthScheme::AwsSigV4 { region, service: SERVICE, .. } if region == expected_region)
);
}
#[rstest]
#[case::default(json!({}), false)]
#[case::override_empty(json!({"claude_platform_unsupported_params": []}), true)]
fn unsupported_fields_are_filtered_before_beta_selection(
#[case] extra: Value,
#[case] keep_context: bool,
) {
let fields = json!({"model": "claude_platform/claude-test", "max_tokens": 16, "messages": [], "workspace_id": "workspace", "aws_region_name": "us-west-2", "context_management": {"edits": [{"type": "clear_thinking_20251015"}]}});
let input = Value::Object(
fields
.as_object()
.unwrap()
.clone()
.into_iter()
.chain(extra.as_object().unwrap().clone())
.collect(),
);
let request = serde_json::from_value(input).unwrap();
let transformed = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
.transform_anthropic_messages_request(request, &MessagesTransformContext::default())
.unwrap();
let headers = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG.request_headers(vec![], &transformed);
let expected_beta = AnthropicBeta::ContextManagement20250627.to_string();
assert_eq!(
header_value(&headers, "anthropic-beta")
.is_some_and(|value| value.split(',').any(|beta| beta == expected_beta)),
keep_context
);
let wire = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
.wire_body(serde_json::to_value(transformed).unwrap());
assert_eq!(wire["model"], "claude-test");
assert_eq!(wire.get("context_management").is_some(), keep_context);
assert_eq!(wire.get("workspace_id"), None);
assert_eq!(wire.get("aws_region_name"), None);
assert_eq!(wire.get(UNSUPPORTED_OVERRIDE), None);
}
}

1
litellm-rust/crates/llms/src/bedrock/claude_platform/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod messages;

16
litellm-rust/crates/llms/src/bedrock/common_utils.rs Normal file
View file

@ -0,0 +1,16 @@
use crate::{Error, ErrorDetail};
pub(crate) fn validated_region(region: String) -> Result<String, Error> {
if !region.is_empty()
&& region
.bytes()
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
{
return Ok(region);
}
Err(Error::InvalidRequest(ErrorDetail::InvalidValue {
field: "aws_region_name",
expected: "lowercase letters, digits, and hyphens",
actual: region.into(),
}))
}

2
litellm-rust/crates/llms/src/bedrock/mod.rs
View file

@ -1,3 +1,5 @@
pub mod audio_transcription;
pub mod chat;
pub mod claude_platform;
pub(crate) mod common_utils;
pub mod messages;

10
litellm-rust/crates/llms/src/bedrock_mantle/messages/AGENTS.md Normal file
View file

@ -0,0 +1,10 @@
# rules
- Reuse Anthropic Messages shaping and reasoning translation
- Use bearer credentials or AWS SigV4 for the Mantle Messages endpoint
- Filter and rename caller betas using the Mantle beta policy
# references
- Python counterpart `litellm/llms/bedrock/messages/mantle_transformation.py`
- https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html

1
litellm-rust/crates/llms/src/bedrock_mantle/messages/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod transformation;

528
litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs Normal file
View file

@ -0,0 +1,528 @@
use std::sync::LazyLock;
use litellm_auth::{AwsParams, CredentialPlacement, SecretValue};
use litellm_auth_aws::AwsCredentialSource;
use litellm_auth_aws::bedrock_model_id_and_region;
use litellm_core_utils::settings::resolve_non_empty;
use litellm_http::request::{with_header, without_headers};
use litellm_llms_types::{
formats::messages::MessagesRequest,
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
};
use litellm_router_types::LitellmParams;
use serde_json::Value;
use crate::{
Error,
anthropic::{
common_utils::{existing_betas, merge_beta_headers},
messages::{
handler::shape_anthropic_messages_request,
transformation::{provider_feature_betas, transform_messages_request},
},
},
base_llm::{
auth::AuthScheme,
messages::{
context::MessagesTransformContext,
transformation::{BaseMessagesConfig, Headers, ValidatedEnvironment},
},
},
bedrock::common_utils::validated_region,
};
const API_BASE_ENV: &str = "BEDROCK_MANTLE_API_BASE";
const REGION_ENVS: &[&str] = &[
"BEDROCK_MANTLE_REGION",
"AWS_REGION_NAME",
"AWS_REGION",
"AWS_DEFAULT_REGION",
];
const KEY_ENVS: &[&str] = &["BEDROCK_MANTLE_API_KEY", "AWS_BEARER_TOKEN_BEDROCK"];
const MESSAGES_PATH: &str = "/anthropic/v1/messages";
const BASE_SUFFIXES: &[&str] = &[
MESSAGES_PATH,
"/v1/messages",
"/messages",
"/anthropic/v1",
"/openai/v1",
"/v1",
];
pub struct BedrockMantleAnthropicMessagesConfig;
pub const BEDROCK_MANTLE_MESSAGES_CONFIG: BedrockMantleAnthropicMessagesConfig =
BedrockMantleAnthropicMessagesConfig;
impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig {
fn validate_environment(
&self,
headers: Headers,
api_key: Option<&str>,
model: &str,
params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
let headers: Headers = headers
.into_iter()
.filter(|(name, _)| {
!name.eq_ignore_ascii_case("authorization")
&& !name.eq_ignore_ascii_case("x-api-key")
})
.collect();
let headers = match params
.extra
.get("aws_bedrock_project_id")
.and_then(Value::as_str)
{
Some(project) => with_header(headers, "anthropic-workspace-id", project.to_string()),
None => headers,
};
let auth = match resolve_non_empty(api_key, env, KEY_ENVS) {
Some(token) => AuthScheme::Credential {
placement: CredentialPlacement::Bearer,
secret: SecretValue::new(token),
},
None => AuthScheme::AwsSigV4 {
region: mantle_region(params.api_base.as_deref(), model, params, env)?,
service: "bedrock",
credentials: Box::new(AwsCredentialSource::from_params(&params.aws, env)),
},
};
Ok(ValidatedEnvironment { headers, auth })
}
fn get_complete_url(
&self,
api_base: Option<&str>,
model: &str,
params: &LitellmParams,
_stream: bool,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let region = mantle_region(api_base, model, params, env)?;
let configured = mantle_base(api_base, params, env)
.unwrap_or_else(|| format!("https://bedrock-mantle.{region}.api.aws"));
let trimmed = configured.trim_end_matches('/');
let stripped = BASE_SUFFIXES
.iter()
.find_map(|suffix| trimmed.strip_suffix(suffix))
.unwrap_or(trimmed);
let base = if mantle_host_region(stripped).is_some() {
format!("https://bedrock-mantle.{region}.api.aws")
} else {
stripped.to_string()
};
Ok(format!("{base}{MESSAGES_PATH}"))
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
transform_messages_request(request, context)
}
fn shape_request(
&self,
request: MessagesRequest,
reasoning_auto_summary: bool,
) -> Result<MessagesRequest, Error> {
shape_anthropic_messages_request(request, reasoning_auto_summary)
}
fn wire_body(&self, body: Value) -> Value {
match body {
Value::Object(fields) => Value::Object(
fields
.into_iter()
.filter_map(|(name, value)| match name.as_str() {
"anthropic_version" | "anthropic_beta" => None,
"model" => Some((
name,
value
.as_str()
.map(|model| Value::String(mantle_model(model).0))
.unwrap_or(value),
)),
_ => Some((name, value)),
})
.collect(),
),
other => other,
}
}
fn secret_names(&self) -> &'static [&'static str] {
static NAMES: LazyLock<Box<[&'static str]>> = LazyLock::new(|| {
[
"BEDROCK_MANTLE_API_KEY",
"AWS_BEARER_TOKEN_BEDROCK",
API_BASE_ENV,
]
.into_iter()
.chain(REGION_ENVS.iter().copied())
.chain(
AwsParams::secret_names()
.iter()
.copied()
.filter(|name| !REGION_ENVS.contains(name)),
)
.collect()
});
&NAMES
}
fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
DEFAULT_HEADERS
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
let body_betas: BetaSet = request
.params
.extra
.get("anthropic_beta")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_str)
.filter_map(|beta| beta.parse::<AnthropicBeta>().ok())
.collect();
let caller: BetaSet = existing_betas(&headers)
.union(body_betas)
.into_iter()
.filter_map(|beta| match beta {
AnthropicBeta::Other(_) => Some(beta),
_ => beta.on(BetaProvider::BedrockMantle),
})
.collect();
let safeguards: BetaSet = request
.params
.extra
.contains_key("safeguards")
.then_some(AnthropicBeta::DangerousToolUse20260903)
.into_iter()
.collect();
merge_beta_headers(
without_headers(headers, &["anthropic-beta"]),
caller
.union(provider_feature_betas(request, BetaProvider::BedrockMantle))
.union(safeguards),
)
}
}
fn mantle_model(model: &str) -> (String, Option<String>) {
bedrock_model_id_and_region(model.strip_prefix("mantle/").unwrap_or(model))
}
fn mantle_host_region(base: &str) -> Option<String> {
let url = url::Url::parse(base).ok()?;
if !matches!(url.scheme(), "https" | "http") {
return None;
}
url.host_str()?
.strip_prefix("bedrock-mantle.")?
.strip_suffix(".api.aws")
.filter(|region| !region.contains('.') && !region.is_empty())
.map(str::to_string)
}
fn mantle_base(
base: Option<&str>,
params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Option<String> {
let explicit = [
base,
params.api_base.as_deref(),
params.aws.aws_bedrock_runtime_endpoint.as_deref(),
]
.into_iter()
.flatten()
.find(|base| !base.trim().is_empty());
resolve_non_empty(explicit, env, &[API_BASE_ENV])
}
fn mantle_region(
base: Option<&str>,
model: &str,
params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let region = mantle_base(base, params, env)
.and_then(|base| mantle_host_region(&base))
.or_else(|| resolve_non_empty(params.aws.aws_region_name.as_deref(), &|_| None, &[]))
.or_else(|| mantle_model(model).1)
.or_else(|| resolve_non_empty(None, env, REGION_ENVS))
.unwrap_or_else(|| "us-east-1".into());
validated_region(region)
}
#[cfg(test)]
mod tests {
use super::*;
use litellm_http::request::header_value;
use rstest::rstest;
use serde_json::json;
#[rstest]
#[case::root("https://private.test", "https://private.test/anthropic/v1/messages")]
#[case::openai(
"https://private.test/openai/v1/",
"https://private.test/anthropic/v1/messages"
)]
#[case::complete(
"https://private.test/anthropic/v1/messages/",
"https://private.test/anthropic/v1/messages"
)]
#[case::tenant(
"https://private.test/tenant/v1",
"https://private.test/tenant/anthropic/v1/messages"
)]
fn endpoints_are_normalized(#[case] base: &str, #[case] expected: &str) {
assert_eq!(
BEDROCK_MANTLE_MESSAGES_CONFIG
.get_complete_url(
Some(base),
"claude-test",
&LitellmParams::default(),
true,
&|_| None
)
.unwrap(),
expected
);
}
#[rstest]
#[case::parameter("us-west-2", "claude-test")]
#[case::model("us-west-2", "us-west-2/claude-test")]
fn private_endpoints_keep_their_host_and_signing_region(
#[case] region: &str,
#[case] model: &str,
) {
let params = LitellmParams {
aws: AwsParams {
aws_region_name: (model == "claude-test").then(|| region.into()),
aws_bedrock_runtime_endpoint: Some("https://private.test/anthropic/v1".into()),
..Default::default()
},
..Default::default()
};
assert_eq!(
BEDROCK_MANTLE_MESSAGES_CONFIG
.get_complete_url(None, model, &params, false, &|_| None)
.unwrap(),
"https://private.test/anthropic/v1/messages"
);
let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
.validate_environment(vec![], None, model, &params, &|_| None)
.unwrap();
assert!(
matches!(environment.auth, AuthScheme::AwsSigV4 { region: actual, service: "bedrock", .. } if actual == region)
);
}
#[rstest]
#[case::explicit_regional_endpoint(
"https://bedrock-mantle.us-west-2.api.aws",
"us-east-1",
"us-west-2"
)]
fn public_regional_host_precedes_configured_region(
#[case] api_base: &str,
#[case] configured_region: &str,
#[case] expected_region: &str,
) {
let params = LitellmParams {
api_base: Some(api_base.into()),
aws: AwsParams {
aws_region_name: Some(configured_region.into()),
..Default::default()
},
..Default::default()
};
let url = BEDROCK_MANTLE_MESSAGES_CONFIG
.get_complete_url(None, "claude-test", &params, false, &|_| None)
.unwrap();
let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
.validate_environment(vec![], None, "claude-test", &params, &|_| None)
.unwrap();
assert_eq!(
url,
"https://bedrock-mantle.us-west-2.api.aws/anthropic/v1/messages"
);
assert!(
matches!(environment.auth, AuthScheme::AwsSigV4 { region, service: "bedrock", .. } if region == expected_region)
);
}
#[rstest]
#[case::aws_default_region("AWS_DEFAULT_REGION", "eu-west-1")]
fn aws_default_region_selects_mantle_endpoint(#[case] env_name: &str, #[case] region: &str) {
let url = BEDROCK_MANTLE_MESSAGES_CONFIG
.get_complete_url(
None,
"claude-test",
&LitellmParams::default(),
false,
&|name| (name == env_name).then(|| region.into()),
)
.unwrap();
assert_eq!(
url,
"https://bedrock-mantle.eu-west-1.api.aws/anthropic/v1/messages"
);
}
#[rstest]
#[case::aws_default_region("AWS_DEFAULT_REGION")]
fn mantle_secret_names_include_region_fallbacks(#[case] env_name: &str) {
assert!(
BEDROCK_MANTLE_MESSAGES_CONFIG
.secret_names()
.contains(&env_name)
);
}
#[rstest]
#[case::explicit(Some("explicit"), "explicit")]
#[case::environment(None, "mantle")]
fn bearer_credentials_replace_forwarded_auth(
#[case] key: Option<&str>,
#[case] expected: &str,
) {
let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
.validate_environment(
vec![
("X-Api-Key".into(), "stale".into()),
("Authorization".into(), "stale".into()),
],
key,
"claude-test",
&LitellmParams::default(),
&|name| match name {
"BEDROCK_MANTLE_API_KEY" => Some("mantle".into()),
"AWS_BEARER_TOKEN_BEDROCK" => Some("aws".into()),
_ => None,
},
)
.unwrap();
assert!(environment.headers.is_empty());
assert!(
matches!(environment.auth, AuthScheme::Credential { placement: CredentialPlacement::Bearer, secret } if secret.expose() == expected)
);
}
#[rstest]
#[case::url_injection("us-east-1.attacker.test/steal?")]
#[case::uppercase("US-EAST-1")]
fn invalid_regions_are_rejected_even_with_bearer_auth(#[case] region: &str) {
let params = LitellmParams {
aws: AwsParams {
aws_region_name: Some(region.into()),
..Default::default()
},
..Default::default()
};
assert!(matches!(
BEDROCK_MANTLE_MESSAGES_CONFIG.get_complete_url(
None,
"claude-test",
&params,
false,
&|_| None
),
Err(Error::InvalidRequest(_))
));
}
#[rstest]
#[case::arn("arn:aws:bedrock:us-east-1.attacker.test:123:model/claude-test", None)]
#[case::environment("claude-test", Some("us-east-1/steal"))]
fn model_arn_and_environment_regions_are_validated(
#[case] model: &str,
#[case] region: Option<&str>,
) {
let url = BEDROCK_MANTLE_MESSAGES_CONFIG.get_complete_url(
None,
model,
&LitellmParams::default(),
false,
&|name| {
(name == "BEDROCK_MANTLE_REGION")
.then(|| region.map(str::to_string))
.flatten()
},
);
assert!(matches!(url, Err(Error::InvalidRequest(_))));
}
#[rstest]
fn caller_betas_are_filtered_renamed_and_merged_with_feature_betas() {
let request: MessagesRequest = serde_json::from_value(json!({"model": "claude-test", "max_tokens": 16, "messages": [], "safeguards": [{"type": "dangerous_tool_use"}]})).unwrap();
let headers = BEDROCK_MANTLE_MESSAGES_CONFIG.request_headers(
vec![(
"Anthropic-Beta".into(),
"advanced-tool-use-2025-11-20,code-execution-2025-08-25,custom-beta".into(),
)],
&request,
);
let actual: std::collections::BTreeSet<&str> = header_value(&headers, "anthropic-beta")
.unwrap()
.split(',')
.collect();
let tool_search = AnthropicBeta::ToolSearchTool20251019.to_string();
let safeguards = AnthropicBeta::DangerousToolUse20260903.to_string();
assert_eq!(
actual,
["custom-beta", tool_search.as_str(), safeguards.as_str()]
.into_iter()
.collect()
);
}
#[rstest]
#[case::body_only(
json!(["custom-beta", "advanced-tool-use-2025-11-20"]),
vec![],
vec![
AnthropicBeta::Other("custom-beta".into()),
AnthropicBeta::ToolSearchTool20251019,
]
)]
#[case::header_and_body(
json!(["custom-b"]),
vec![("Anthropic-Beta".into(), "custom-a".into())],
vec![
AnthropicBeta::Other("custom-a".into()),
AnthropicBeta::Other("custom-b".into()),
]
)]
fn body_betas_are_filtered_renamed_and_merged_with_headers(
#[case] body_betas: Value,
#[case] headers: Headers,
#[case] expected: Vec<AnthropicBeta>,
) {
let request: MessagesRequest = serde_json::from_value(json!({
"model": "claude-test",
"max_tokens": 16,
"messages": [],
"anthropic_beta": body_betas
}))
.unwrap();
let headers = BEDROCK_MANTLE_MESSAGES_CONFIG.request_headers(headers, &request);
let actual: BetaSet = header_value(&headers, "anthropic-beta")
.unwrap()
.parse()
.unwrap();
let expected: BetaSet = expected.into_iter().collect();
assert_eq!(actual, expected);
}
#[rstest]
fn wire_body_keeps_streaming_and_removes_sdk_fields() {
assert_eq!(BEDROCK_MANTLE_MESSAGES_CONFIG.wire_body(json!({"model": "mantle/us-west-2/anthropic.claude-test", "stream": true, "anthropic_version": "sdk", "anthropic_beta": ["sdk"], "messages": []})), json!({"model": "anthropic.claude-test", "stream": true, "messages": []}));
}
}

1
litellm-rust/crates/llms/src/bedrock_mantle/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod messages;

1
litellm-rust/crates/llms/src/edenai/messages/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod transformation;

120
litellm-rust/crates/llms/src/edenai/messages/transformation.rs Normal file
View file

@ -0,0 +1,120 @@
use litellm_auth::{CredentialPlacement, SecretValue};
use litellm_core_utils::settings::resolve_non_empty;
use litellm_llms_types::formats::messages::{
MessagesOptionalParams, MessagesRequest, MessagesResponse,
};
use litellm_llms_types::providers::anthropic::DEFAULT_HEADERS;
use litellm_router_types::LitellmParams;
use serde_json::Value;
use crate::{
Error,
anthropic::{
common_utils::{filter_billing_headers_from_system, has_anthropic_credential},
messages::transformation::update_headers_with_anthropic_beta,
},
base_llm::{
auth::AuthScheme,
messages::{
context::MessagesTransformContext,
transformation::{BaseMessagesConfig, Headers, ValidatedEnvironment},
},
},
openai_like::messages::transformation::{complete_messages_url, portable_cache_control},
};
const API_KEY_ENV: &str = "EDENAI_API_KEY";
const API_BASE_ENV: &str = "EDENAI_API_BASE";
const DEFAULT_BASE: &str = "https://api.edenai.run/v3";
pub struct EdenAIAnthropicMessagesConfig;
pub const EDENAI_MESSAGES_CONFIG: EdenAIAnthropicMessagesConfig = EdenAIAnthropicMessagesConfig;
impl BaseMessagesConfig for EdenAIAnthropicMessagesConfig {
fn validate_environment(
&self,
headers: Headers,
api_key: Option<&str>,
_model: &str,
_params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
let key = resolve_non_empty(api_key, env, &[API_KEY_ENV]).ok_or(Error::Auth(
litellm_auth::Error::MissingApiKey {
provider: "Eden AI",
environment_variable: API_KEY_ENV,
},
))?;
if has_anthropic_credential(&headers) {
return Ok(ValidatedEnvironment {
headers,
auth: AuthScheme::Forwarded,
});
}
Ok(ValidatedEnvironment {
headers,
auth: AuthScheme::Credential {
placement: CredentialPlacement::Bearer,
secret: SecretValue::new(key),
},
})
}
fn get_complete_url(
&self,
api_base: Option<&str>,
_model: &str,
_params: &LitellmParams,
_stream: bool,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let base = resolve_non_empty(api_base, env, &[API_BASE_ENV])
.unwrap_or_else(|| DEFAULT_BASE.into());
Ok(complete_messages_url(&base))
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
_context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
Ok(MessagesRequest {
params: MessagesOptionalParams {
system: request
.params
.system
.and_then(filter_billing_headers_from_system),
..request.params
},
..request
})
}
fn secret_names(&self) -> &'static [&'static str] {
&[API_KEY_ENV, API_BASE_ENV]
}
fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
DEFAULT_HEADERS
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
}
fn wire_body(&self, body: Value) -> Value {
portable_cache_control(body)
}
fn reported_cost(&self, response: &MessagesResponse) -> Option<serde_json::Number> {
let cost = match response.extra.get("cost")? {
Value::Number(cost) => cost.as_f64()?,
Value::String(cost) => cost.parse::<f64>().ok()?,
_ => return None,
};
if cost < 0.0 {
return None;
}
serde_json::Number::from_f64(cost)
}
}

1
litellm-rust/crates/llms/src/edenai/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod messages;

5
litellm-rust/crates/llms/src/lib.rs
View file

@ -3,13 +3,18 @@ pub mod aws_textract;
pub mod azure_ai;
pub mod base_llm;
pub mod bedrock;
pub mod bedrock_mantle;
pub mod cohere;
pub mod deepseek;
pub mod edenai;
mod error;
pub mod minimax;
pub mod mistral;
pub mod openai;
pub mod openai_like;
pub mod openrouter;
pub mod reducto;
pub mod tencent;
pub mod vertex_ai;
pub use error::{Error, ErrorDetail, ErrorSource};

9
litellm-rust/crates/llms/src/minimax/messages/AGENTS.md Normal file
View file

@ -0,0 +1,9 @@
# rules
- Reuse Anthropic Messages shaping and reasoning translation
- Use MiniMax credentials and endpoints, and strip billing system blocks
# references
- Python counterpart `litellm/llms/minimax/messages/transformation.py`
- Upstream contract in `llms-types/src/providers/minimax/AGENTS.md`

1
litellm-rust/crates/llms/src/minimax/messages/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod transformation;

195
litellm-rust/crates/llms/src/minimax/messages/transformation.rs Normal file
View file

@ -0,0 +1,195 @@
use litellm_auth::{CredentialPlacement, SecretValue};
use litellm_core_utils::settings::resolve_non_empty;
use litellm_llms_types::{
formats::messages::{MessagesOptionalParams, MessagesRequest},
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
use crate::{
Error,
anthropic::{
common_utils::{filter_billing_headers_from_system, has_anthropic_credential},
messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
},
},
base_llm::{
auth::{AuthScheme, Headers, ValidatedEnvironment},
messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig},
},
};
const API_KEY_ENV: &str = "MINIMAX_API_KEY";
const API_BASE_ENV: &str = "MINIMAX_API_BASE";
const DEFAULT_BASE: &str = "https://api.minimax.io/anthropic";
pub struct MinimaxMessagesConfig;
pub const MINIMAX_MESSAGES_CONFIG: MinimaxMessagesConfig = MinimaxMessagesConfig;
impl BaseMessagesConfig for MinimaxMessagesConfig {
fn shape_request(
&self,
request: MessagesRequest,
reasoning_auto_summary: bool,
) -> Result<MessagesRequest, Error> {
shape_anthropic_messages_request(request, reasoning_auto_summary)
}
fn validate_environment(
&self,
headers: Headers,
api_key: Option<&str>,
_model: &str,
_params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
if has_anthropic_credential(&headers) {
return Ok(ValidatedEnvironment {
headers,
auth: AuthScheme::Forwarded,
});
}
let key = resolve_non_empty(api_key, env, &[API_KEY_ENV]).ok_or(Error::Auth(
litellm_auth::Error::MissingApiKey {
provider: "MiniMax",
environment_variable: API_KEY_ENV,
},
))?;
Ok(ValidatedEnvironment {
headers,
auth: AuthScheme::Credential {
placement: CredentialPlacement::Header("x-api-key"),
secret: SecretValue::new(key),
},
})
}
fn get_complete_url(
&self,
api_base: Option<&str>,
_model: &str,
_params: &LitellmParams,
_stream: bool,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let base = resolve_non_empty(api_base, env, &[API_BASE_ENV])
.unwrap_or_else(|| DEFAULT_BASE.into());
let base = base.trim_end_matches('/');
if base.ends_with(MESSAGES_PATH) {
return Ok(base.into());
}
Ok(format!("{base}{MESSAGES_PATH}"))
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
let request = transform_messages_request(request, context)?;
Ok(MessagesRequest {
params: MessagesOptionalParams {
system: request
.params
.system
.and_then(filter_billing_headers_from_system),
..request.params
},
..request
})
}
fn secret_names(&self) -> &'static [&'static str] {
&[API_KEY_ENV, API_BASE_ENV]
}
fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
DEFAULT_HEADERS
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
fn no_env(_: &str) -> Option<String> {
None
}
#[rstest]
#[case::root("https://proxy.test", "https://proxy.test/v1/messages")]
#[case::root_slash("https://proxy.test/", "https://proxy.test/v1/messages")]
#[case::full_endpoint("https://proxy.test/v1/messages", "https://proxy.test/v1/messages")]
#[case::full_endpoint_slash(
"https://proxy.test/v1/messages/",
"https://proxy.test/v1/messages"
)]
#[case::custom_path(
"https://proxy.test/anthropic",
"https://proxy.test/anthropic/v1/messages"
)]
fn endpoint_is_appended_once(#[case] base: &str, #[case] expected: &str) {
assert_eq!(
MINIMAX_MESSAGES_CONFIG.get_complete_url(
Some(base),
"test-model",
&Default::default(),
false,
&no_env,
),
Ok(expected.into())
);
}
#[rstest]
#[case::explicit(Some("https://explicit.test"), "https://explicit.test/v1/messages")]
#[case::environment(None, "https://env.test/v1/messages")]
#[case::blank_explicit(Some(" "), "https://env.test/v1/messages")]
fn base_precedence(#[case] base: Option<&str>, #[case] expected: &str) {
let env =
|name: &str| (name == API_BASE_ENV).then(|| "https://env.test/v1/messages/".into());
assert_eq!(
MINIMAX_MESSAGES_CONFIG.get_complete_url(
base,
"test-model",
&Default::default(),
false,
&env,
),
Ok(expected.into())
);
}
#[rstest]
#[case::explicit(Some("explicit"), Some("environment"), "explicit")]
#[case::environment(None, Some("environment"), "environment")]
#[case::blank_explicit(Some(" "), Some("environment"), "environment")]
fn key_precedence(
#[case] key: Option<&str>,
#[case] env_key: Option<&str>,
#[case] expected: &str,
) {
let env = |name: &str| {
(name == API_KEY_ENV)
.then(|| env_key.map(str::to_owned))
.flatten()
};
let environment = MINIMAX_MESSAGES_CONFIG
.validate_environment(vec![], key, "test-model", &Default::default(), &env)
.unwrap();
assert!(matches!(
environment.auth,
AuthScheme::Credential {
placement: CredentialPlacement::Header("x-api-key"), ref secret
} if secret.expose() == expected
));
}
}

1
litellm-rust/crates/llms/src/minimax/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod messages;

1
litellm-rust/crates/llms/src/openai_like/messages/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod transformation;

100
litellm-rust/crates/llms/src/openai_like/messages/transformation.rs Normal file
View file

@ -0,0 +1,100 @@
use serde_json::{Map, Value};
pub fn complete_messages_url(base: &str) -> String {
let base = base.trim_end_matches('/');
if base.ends_with("/v1/messages") {
return base.into();
}
format!("{}/v1/messages", base.strip_suffix("/v1").unwrap_or(base))
}
pub fn portable_cache_control(body: Value) -> Value {
let Value::Object(fields) = body else {
return body;
};
Value::Object(
portable_block(fields)
.into_iter()
.map(|(key, value)| {
let value = match key.as_str() {
"system" | "tools" => map_blocks(value, portable_value),
"messages" => map_blocks(value, portable_message),
_ => value,
};
(key, value)
})
.collect(),
)
}
fn portable_block(fields: Map<String, Value>) -> Map<String, Value> {
fields
.into_iter()
.filter_map(|(key, value)| {
if key != "cache_control" {
return Some((key, value));
}
let Value::Object(cache) = value else {
return None;
};
let kind = cache
.get("type")
.and_then(Value::as_str)
.unwrap_or("ephemeral");
Some((key, serde_json::json!({"type": kind})))
})
.collect()
}
fn portable_value(value: Value) -> Value {
match value {
Value::Object(fields) => Value::Object(portable_block(fields)),
other => other,
}
}
fn map_blocks(value: Value, map: fn(Value) -> Value) -> Value {
match value {
Value::Array(blocks) => Value::Array(blocks.into_iter().map(map).collect()),
other => other,
}
}
fn portable_message(value: Value) -> Value {
let Value::Object(fields) = value else {
return value;
};
Value::Object(
fields
.into_iter()
.map(|(key, value)| {
let value = if key == "content" {
map_blocks(value, portable_content)
} else {
value
};
(key, value)
})
.collect(),
)
}
fn portable_content(value: Value) -> Value {
let Value::Object(fields) = value else {
return value;
};
let tool_result = fields.get("type").and_then(Value::as_str) == Some("tool_result");
Value::Object(
portable_block(fields)
.into_iter()
.map(|(key, value)| {
let value = if tool_result && key == "content" {
map_blocks(value, portable_value)
} else {
value
};
(key, value)
})
.collect(),
)
}

1
litellm-rust/crates/llms/src/openai_like/mod.rs
View file

@ -1,2 +1,3 @@
pub mod chat;
pub mod common_utils;
pub mod messages;

10
litellm-rust/crates/llms/src/openrouter/messages/AGENTS.md Normal file
View file

@ -0,0 +1,10 @@
# rules
- Send the native Messages payload to `/api/v1/messages` and relay Anthropic SSE
- Authenticate with a bearer token, never treat `x-api-key` as an OpenRouter credential
- Keep cache TTLs, routing fields, metadata, and provider usage extensions
- Strip first-party billing system blocks using the Anthropic helper
# references
- https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-messages

1
litellm-rust/crates/llms/src/openrouter/messages/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod transformation;

198
litellm-rust/crates/llms/src/openrouter/messages/transformation.rs Normal file
View file

@ -0,0 +1,198 @@
use litellm_auth::{CredentialPlacement, SecretValue};
use litellm_core_utils::settings::resolve_non_empty;
use litellm_http::request::{has_bearer_auth, without_headers};
use litellm_llms_types::{
formats::messages::{MessagesOptionalParams, MessagesRequest},
providers::anthropic::DEFAULT_HEADERS,
};
use litellm_router_types::LitellmParams;
use crate::{
Error,
anthropic::{
common_utils::filter_billing_headers_from_system,
messages::transformation::update_headers_with_anthropic_beta,
},
base_llm::{
auth::{AuthScheme, Headers, ValidatedEnvironment},
messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig},
},
};
const API_KEY_ENV: &str = "OPENROUTER_API_KEY";
const API_BASE_ENV: &str = "OPENROUTER_API_BASE";
const DEFAULT_BASE: &str = "https://openrouter.ai/api/v1";
const MESSAGES_PATH: &str = "/v1/messages";
pub struct OpenRouterAnthropicMessagesConfig;
pub const OPENROUTER_MESSAGES_CONFIG: OpenRouterAnthropicMessagesConfig =
OpenRouterAnthropicMessagesConfig;
impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig {
fn validate_environment(
&self,
headers: Headers,
api_key: Option<&str>,
_model: &str,
_params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
let headers = without_headers(headers, &["x-api-key"]);
if has_bearer_auth(&headers) {
return Ok(ValidatedEnvironment {
headers,
auth: AuthScheme::Forwarded,
});
}
let key = resolve_non_empty(api_key, env, &[API_KEY_ENV]).ok_or(Error::Auth(
litellm_auth::Error::MissingApiKey {
provider: "OpenRouter",
environment_variable: API_KEY_ENV,
},
))?;
Ok(ValidatedEnvironment {
headers: without_headers(headers, &["authorization"]),
auth: AuthScheme::Credential {
placement: CredentialPlacement::Bearer,
secret: SecretValue::new(key),
},
})
}
fn get_complete_url(
&self,
api_base: Option<&str>,
_model: &str,
params: &LitellmParams,
_stream: bool,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let explicit = api_base
.filter(|base| !base.trim().is_empty())
.or(params.api_base.as_deref());
let base = resolve_non_empty(explicit, env, &[API_BASE_ENV])
.unwrap_or_else(|| DEFAULT_BASE.into());
let trimmed = base.trim_end_matches('/');
if trimmed.ends_with(MESSAGES_PATH) {
return Ok(trimmed.into());
}
Ok(format!(
"{}{MESSAGES_PATH}",
trimmed.strip_suffix("/v1").unwrap_or(trimmed)
))
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
_context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
Ok(MessagesRequest {
params: MessagesOptionalParams {
system: request
.params
.system
.and_then(filter_billing_headers_from_system),
..request.params
},
..request
})
}
fn secret_names(&self) -> &'static [&'static str] {
&[API_KEY_ENV, API_BASE_ENV]
}
fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
DEFAULT_HEADERS
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
}
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
use serde_json::json;
#[rstest]
#[case::default(None, "https://openrouter.ai/api/v1/messages")]
#[case::root(Some("https://proxy.test"), "https://proxy.test/v1/messages")]
#[case::version(
Some("https://proxy.test/api/v1/"),
"https://proxy.test/api/v1/messages"
)]
#[case::complete(
Some("https://proxy.test/api/v1/messages/"),
"https://proxy.test/api/v1/messages"
)]
fn urls_append_messages_once(#[case] base: Option<&str>, #[case] expected: &str) {
assert_eq!(
OPENROUTER_MESSAGES_CONFIG
.get_complete_url(
base,
"anthropic/test-model",
&LitellmParams::default(),
true,
&|_| None
)
.unwrap(),
expected
);
}
#[rstest]
#[case::call(
Some("https://call.test/v1"),
Some("https://deployment.test/v1"),
"https://call.test/v1/messages"
)]
#[case::deployment(
None,
Some("https://deployment.test/v1"),
"https://deployment.test/v1/messages"
)]
#[case::blank_call(
Some(" "),
Some("https://deployment.test/v1"),
"https://deployment.test/v1/messages"
)]
#[case::environment(None, None, "https://secret.test/v1/messages")]
fn base_precedence_is_call_deployment_then_secrets(
#[case] base: Option<&str>,
#[case] deployment: Option<&str>,
#[case] expected: &str,
) {
let params = LitellmParams {
api_base: deployment.map(str::to_string),
..Default::default()
};
assert_eq!(
OPENROUTER_MESSAGES_CONFIG
.get_complete_url(base, "test-model", &params, false, &|name| (name
== API_BASE_ENV)
.then(|| "https://secret.test/v1".into()))
.unwrap(),
expected
);
}
#[rstest]
fn native_payload_is_preserved_except_first_party_billing_blocks() {
let input = json!({"model": "anthropic/test-model", "max_tokens": 128, "messages": [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}}]}], "system": [{"type": "text", "text": "x-anthropic-billing-header: private"}, {"type": "text", "text": "instructions"}], "metadata": {"user_id": "user", "custom": "keep"}, "provider": {"order": ["test-provider"]}, "thinking": {"type": "enabled", "budget_tokens": 64}});
let request = serde_json::from_value(input.clone()).unwrap();
let actual = OPENROUTER_MESSAGES_CONFIG
.transform_anthropic_messages_request(request, &MessagesTransformContext::default())
.unwrap();
let expected = json!({"system": [{"type": "text", "text": "instructions"}]});
let body = serde_json::to_value(actual).unwrap();
assert_eq!(body["system"], expected["system"]);
assert_eq!(body["messages"], input["messages"]);
assert_eq!(body["metadata"], input["metadata"]);
assert_eq!(body["provider"], input["provider"]);
assert_eq!(body["thinking"], input["thinking"]);
}
}

1
litellm-rust/crates/llms/src/openrouter/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod messages;

11
litellm-rust/crates/llms/src/tencent/messages/AGENTS.md Normal file
View file

@ -0,0 +1,11 @@
# rules
- Reuse Anthropic Messages shaping, reasoning translation, and history cleanup
- Use Tencent credentials and endpoints, and strip billing system blocks
- Accept a root, versioned base, chat endpoint, or complete Messages endpoint
# references
- Python counterpart `litellm/llms/tencent/messages/transformation.py`
- https://www.tencentcloud.com/document/product/1300/80632
- https://cloud.tencent.com/document/product/1823/132246

1
litellm-rust/crates/llms/src/tencent/messages/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod transformation;

241
litellm-rust/crates/llms/src/tencent/messages/transformation.rs Normal file
View file

@ -0,0 +1,241 @@
use litellm_auth::{CredentialPlacement, SecretValue};
use litellm_core_utils::settings::resolve_non_empty;
use litellm_llms_types::{
formats::messages::{MessagesOptionalParams, MessagesRequest},
providers::anthropic::{DEFAULT_HEADERS, MESSAGES_PATH},
};
use litellm_router_types::LitellmParams;
use crate::{
Error,
anthropic::{
common_utils::{filter_billing_headers_from_system, has_anthropic_credential},
messages::{
handler::shape_anthropic_messages_request,
transformation::{transform_messages_request, update_headers_with_anthropic_beta},
},
},
base_llm::{
auth::{AuthScheme, Headers, ValidatedEnvironment},
messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig},
},
};
const API_KEY_ENV: &str = "TENCENT_API_KEY";
const API_BASE_ENV: &str = "TENCENT_API_BASE";
const ANTHROPIC_API_BASE_ENV: &str = "TENCENT_ANTHROPIC_API_BASE";
const DEFAULT_BASE: &str = "https://tokenhub-intl.tencentcloudmaas.com";
pub struct TencentAnthropicMessagesConfig;
pub const TENCENT_MESSAGES_CONFIG: TencentAnthropicMessagesConfig = TencentAnthropicMessagesConfig;
impl BaseMessagesConfig for TencentAnthropicMessagesConfig {
fn shape_request(
&self,
request: MessagesRequest,
reasoning_auto_summary: bool,
) -> Result<MessagesRequest, Error> {
shape_anthropic_messages_request(request, reasoning_auto_summary)
}
fn validate_environment(
&self,
headers: Headers,
api_key: Option<&str>,
_model: &str,
_params: &LitellmParams,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error> {
if has_anthropic_credential(&headers) {
return Ok(ValidatedEnvironment {
headers,
auth: AuthScheme::Forwarded,
});
}
let key = resolve_non_empty(api_key, env, &[API_KEY_ENV]).ok_or(Error::Auth(
litellm_auth::Error::MissingApiKey {
provider: "Tencent",
environment_variable: API_KEY_ENV,
},
))?;
Ok(ValidatedEnvironment {
headers,
auth: AuthScheme::Credential {
placement: CredentialPlacement::Header("x-api-key"),
secret: SecretValue::new(key),
},
})
}
fn get_complete_url(
&self,
api_base: Option<&str>,
_model: &str,
_params: &LitellmParams,
_stream: bool,
env: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let base = resolve_non_empty(api_base, env, &[ANTHROPIC_API_BASE_ENV, API_BASE_ENV])
.unwrap_or_else(|| DEFAULT_BASE.into());
let base = base.trim_end_matches('/');
if base.ends_with(MESSAGES_PATH) {
return Ok(base.into());
}
let base = base
.strip_suffix("/v1/chat/completions")
.or_else(|| base.strip_suffix("/v1"))
.unwrap_or(base);
Ok(format!("{base}{MESSAGES_PATH}"))
}
fn transform_anthropic_messages_request(
&self,
request: MessagesRequest,
context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
let request = transform_messages_request(request, context)?;
Ok(MessagesRequest {
params: MessagesOptionalParams {
system: request
.params
.system
.and_then(filter_billing_headers_from_system),
..request.params
},
..request
})
}
fn secret_names(&self) -> &'static [&'static str] {
&[API_KEY_ENV, ANTHROPIC_API_BASE_ENV, API_BASE_ENV]
}
fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
DEFAULT_HEADERS
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
update_headers_with_anthropic_beta(headers, request)
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
fn no_env(_: &str) -> Option<String> {
None
}
#[rstest]
#[case::root("https://proxy.test", "https://proxy.test/v1/messages")]
#[case::root_slash("https://proxy.test/", "https://proxy.test/v1/messages")]
#[case::full_endpoint("https://proxy.test/v1/messages", "https://proxy.test/v1/messages")]
#[case::full_endpoint_slash(
"https://proxy.test/v1/messages/",
"https://proxy.test/v1/messages"
)]
#[case::custom_path(
"https://proxy.test/anthropic",
"https://proxy.test/anthropic/v1/messages"
)]
#[case::versioned("https://proxy.test/v1", "https://proxy.test/v1/messages")]
#[case::chat_endpoint(
"https://proxy.test/v1/chat/completions/",
"https://proxy.test/v1/messages"
)]
fn endpoint_is_appended_once(#[case] base: &str, #[case] expected: &str) {
assert_eq!(
TENCENT_MESSAGES_CONFIG.get_complete_url(
Some(base),
"test-model",
&Default::default(),
false,
&no_env,
),
Ok(expected.into())
);
}
#[rstest]
#[case::default(None, None, None, &format!("{DEFAULT_BASE}{MESSAGES_PATH}"))]
#[case::explicit(
Some("https://explicit.test"),
Some("https://anthropic.test"),
Some("https://env.test"),
"https://explicit.test/v1/messages"
)]
#[case::anthropic_base(
None,
Some("https://anthropic.test"),
Some("https://env.test"),
"https://anthropic.test/v1/messages"
)]
#[case::generic_base(
None,
None,
Some("https://env.test/v1"),
"https://env.test/v1/messages"
)]
#[case::blank_anthropic_base(
None,
Some(" "),
Some("https://env.test"),
"https://env.test/v1/messages"
)]
#[case::blank_explicit(
Some(" "),
Some("https://anthropic.test/v1/messages/"),
None,
"https://anthropic.test/v1/messages"
)]
fn base_precedence(
#[case] base: Option<&str>,
#[case] anthropic_base: Option<&str>,
#[case] generic_base: Option<&str>,
#[case] expected: &str,
) {
let env = |name: &str| match name {
ANTHROPIC_API_BASE_ENV => anthropic_base.map(str::to_owned),
API_BASE_ENV => generic_base.map(str::to_owned),
_ => None,
};
assert_eq!(
TENCENT_MESSAGES_CONFIG.get_complete_url(
base,
"test-model",
&Default::default(),
false,
&env,
),
Ok(expected.into())
);
}
#[rstest]
#[case::explicit(Some("explicit"), Some("environment"), "explicit")]
#[case::environment(None, Some("environment"), "environment")]
#[case::blank_explicit(Some(" "), Some("environment"), "environment")]
fn key_precedence(
#[case] key: Option<&str>,
#[case] env_key: Option<&str>,
#[case] expected: &str,
) {
let env = |name: &str| {
(name == API_KEY_ENV)
.then(|| env_key.map(str::to_owned))
.flatten()
};
let environment = TENCENT_MESSAGES_CONFIG
.validate_environment(vec![], key, "test-model", &Default::default(), &env)
.unwrap();
assert!(matches!(
environment.auth,
AuthScheme::Credential {
placement: CredentialPlacement::Header("x-api-key"), ref secret
} if secret.expose() == expected
));
}
}

1
litellm-rust/crates/llms/src/tencent/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod messages;

34
litellm-rust/crates/llms/tests/edenai_messages_transformation.rs Normal file
View file

@ -0,0 +1,34 @@
use litellm_llms::{
base_llm::messages::transformation::BaseMessagesConfig,
edenai::messages::transformation::EDENAI_MESSAGES_CONFIG,
};
use litellm_llms_types::formats::messages::MessagesResponse;
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::number(json!(0.025), Some(0.025))]
#[case::free(json!(0), Some(0.0))]
#[case::text(json!("0.75"), Some(0.75))]
#[case::absent(Value::Null, None)]
#[case::malformed(json!("not-a-cost"), None)]
#[case::negative(json!(-1), None)]
#[case::boolean(json!(true), None)]
#[case::infinite(json!("inf"), None)]
#[case::nan(json!("NaN"), None)]
fn reported_cost_is_a_finite_nonnegative_amount(
#[case] cost: Value,
#[case] expected: Option<f64>,
) {
let response: MessagesResponse = serde_json::from_value(json!({
"id": "msg_test", "type": "message", "role": "assistant", "model": "native-test",
"content": [], "stop_reason": "end_turn", "stop_sequence": null, "cost": cost,
}))
.unwrap();
assert_eq!(
EDENAI_MESSAGES_CONFIG
.reported_cost(&response)
.and_then(|cost| cost.as_f64()),
expected
);
}

64
litellm-rust/crates/llms/tests/openai_like_messages_transformation.rs Normal file
View file

@ -0,0 +1,64 @@
use litellm_llms::openai_like::messages::transformation::{
complete_messages_url, portable_cache_control,
};
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::root("https://native.test", "https://native.test/v1/messages")]
#[case::trailing_slash("https://native.test/", "https://native.test/v1/messages")]
#[case::versioned("https://native.test/v1/", "https://native.test/v1/messages")]
#[case::full_url(
"https://native.test/v3/v1/messages/",
"https://native.test/v3/v1/messages"
)]
#[case::gateway_prefix("https://native.test/v3", "https://native.test/v3/v1/messages")]
fn messages_url_appends_the_endpoint_once(#[case] base: &str, #[case] expected: &str) {
assert_eq!(complete_messages_url(base), expected);
}
#[rstest]
#[case::extended(json!({"type": "ephemeral", "ttl": "1h", "scope": "global"}), Some(json!({"type": "ephemeral"})))]
#[case::missing_type(json!({"ttl": "1h"}), Some(json!({"type": "ephemeral"})))]
#[case::invalid_type(json!({"type": 7}), Some(json!({"type": "ephemeral"})))]
#[case::null(Value::Null, None)]
#[case::scalar(json!(false), None)]
fn portable_cache_only_rewrites_protocol_cache_hints(
#[case] cache: Value,
#[case] expected_cache: Option<Value>,
) {
let application_data = json!({"cache_control": cache, "nested": {"cache_control": cache}});
let body = json!({
"cache_control": cache,
"model": "native-test",
"system": [{"type": "text", "text": "system", "cache_control": cache}],
"tools": [{"name": "tool", "cache_control": cache, "input_schema": application_data}],
"messages": [{"role": "user", "content": [
{"type": "tool_use", "input": application_data, "cache_control": cache},
{"type": "tool_result", "content": [{"type": "text", "text": "result", "cache_control": cache}], "cache_control": cache},
]}],
});
let hint = |value: Value| {
let fields = value.as_object().unwrap().clone();
Value::Object(
fields
.into_iter()
.chain(
expected_cache
.clone()
.map(|cache| ("cache_control".into(), cache)),
)
.collect(),
)
};
let expected = hint(json!({
"model": "native-test",
"system": [hint(json!({"type": "text", "text": "system"}))],
"tools": [hint(json!({"name": "tool", "input_schema": application_data}))],
"messages": [{"role": "user", "content": [
hint(json!({"type": "tool_use", "input": application_data})),
hint(json!({"type": "tool_result", "content": [hint(json!({"type": "text", "text": "result"}))]})),
]}],
}));
assert_eq!(portable_cache_control(body), expected);
}

2
litellm-rust/crates/python-bridge/src/cache/AGENTS.md vendored
View file

@ -2,6 +2,6 @@
This folder owns how Rust inference reaches the selected cache: global cache selection, route admission, delegation to a Python cache and the experimental V2 native handles. Cache algorithms, storage protocols and response-cache semantics belong to their cache crates
`mod.rs` exposes the cache boundary to routes and module registration. Adapter directories remain private. `selection.rs` owns global cache selection, route admission and inference protocol composition for both adapters
`mod.rs` exposes the cache boundary to routes and module registration. Adapter directories remain private. `selection.rs` asks `litellm.rust_bridge.host.cache` for the live configured cache, and owns route admission and inference protocol composition for both adapters
`python/` delegates operations to the selected Python cache without discovering configuration. `native/` owns the experimental V2 native cache handles. Neither adapter depends on shared selection or the other adapter. Shared composition depends on the adapters, and routes use only the parent module's exports

7
litellm-rust/crates/python-bridge/src/cache/selection.rs vendored
View file

@ -9,6 +9,8 @@ use litellm_host::{
use pyo3::{prelude::*, types::PyDict};
use std::sync::Arc;
const CACHE_HOST_MODULE: &str = "litellm.rust_bridge.host.cache";
pub(crate) struct Cached<P>(std::marker::PhantomData<P>);
impl<P: Protocol> Protocol for Cached<P> {
@ -56,7 +58,10 @@ fn selected_cache<'py>(
kwargs: &Bound<'py, PyDict>,
call_type: &str,
) -> PyResult<Option<Bound<'py, PyAny>>> {
let configured = py.import("litellm")?.getattr("cache")?;
let configured = py
.import(CACHE_HOST_MODULE)?
.getattr("configured_cache")?
.call0()?;
if configured.is_none()
|| kwargs
.get_item("caching")?

4
litellm-rust/crates/python-bridge/src/marshal.rs
View file

@ -9,6 +9,8 @@ use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
use serde::{Serialize, de::DeserializeOwned};
use serde_json::{Map, Value};
const TIMEOUTS_MODULE: &str = "litellm.rust_bridge.host.timeouts";
/// The keyword arguments every value route shares, validated at the Python boundary.
pub(crate) struct RouteOptions {
pub(crate) model: String,
@ -47,7 +49,7 @@ pub(crate) fn optional_timeout(timeout_seconds: Option<f64>) -> Option<Duration>
}
pub(crate) fn python_timeout_seconds(py: Python<'_>, timeout: Py<PyAny>) -> PyResult<Option<f64>> {
py.import("litellm.rust_bridge.timeouts")?
py.import(TIMEOUTS_MODULE)?
.getattr("timeout_to_seconds")?
.call1((timeout,))?
.extract()

30
litellm-rust/crates/python-bridge/src/python_settings.rs
View file

@ -3,7 +3,7 @@ use strum::IntoStaticStr;
use crate::coercion::{FieldSpec, ProjectionError};
const MODULE: &str = "litellm.rust_bridge.settings";
const MODULE: &str = "litellm.rust_bridge.host.settings";
#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq)]
pub(crate) enum PythonSettings {
@ -220,22 +220,26 @@ import sys
import types
previous_modules = {
name: sys.modules[name]
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.settings')
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host', 'litellm.rust_bridge.host.settings')
if name in sys.modules
}
litellm = types.ModuleType('litellm')
litellm.__path__ = []
rust_bridge = types.ModuleType('litellm.rust_bridge')
rust_bridge.__path__ = []
settings = types.ModuleType('litellm.rust_bridge.settings')
host = types.ModuleType('litellm.rust_bridge.host')
host.__path__ = []
settings = types.ModuleType('litellm.rust_bridge.host.settings')
def http_settings():
raise ModuleNotFoundError('No module named certifi', name='certifi')
settings.http_settings = http_settings
litellm.rust_bridge = rust_bridge
rust_bridge.settings = settings
rust_bridge.host = host
host.settings = settings
sys.modules['litellm'] = litellm
sys.modules['litellm.rust_bridge'] = rust_bridge
sys.modules['litellm.rust_bridge.settings'] = settings
sys.modules['litellm.rust_bridge.host'] = host
sys.modules['litellm.rust_bridge.host.settings'] = settings
",
Some(&locals),
Some(&locals),
@ -257,7 +261,7 @@ sys.modules['litellm.rust_bridge.settings'] = settings
);
py.run(
c"
for name in ('litellm.rust_bridge.settings', 'litellm.rust_bridge', 'litellm'):
for name in ('litellm.rust_bridge.host.settings', 'litellm.rust_bridge.host', 'litellm.rust_bridge', 'litellm'):
sys.modules.pop(name, None)
sys.modules.update(previous_modules)
",
@ -279,22 +283,26 @@ import sys
import types
previous_modules = {
name: sys.modules[name]
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.settings')
for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host', 'litellm.rust_bridge.host.settings')
if name in sys.modules
}
litellm = types.ModuleType('litellm')
litellm.__path__ = []
rust_bridge = types.ModuleType('litellm.rust_bridge')
rust_bridge.__path__ = []
settings = types.ModuleType('litellm.rust_bridge.settings')
host = types.ModuleType('litellm.rust_bridge.host')
host.__path__ = []
settings = types.ModuleType('litellm.rust_bridge.host.settings')
def http_settings():
raise ImportError('cannot import name setting')
settings.http_settings = http_settings
litellm.rust_bridge = rust_bridge
rust_bridge.settings = settings
rust_bridge.host = host
host.settings = settings
sys.modules['litellm'] = litellm
sys.modules['litellm.rust_bridge'] = rust_bridge
sys.modules['litellm.rust_bridge.settings'] = settings
sys.modules['litellm.rust_bridge.host'] = host
sys.modules['litellm.rust_bridge.host.settings'] = settings
",
Some(&locals),
Some(&locals),
@ -308,7 +316,7 @@ sys.modules['litellm.rust_bridge.settings'] = settings
assert_eq!(error.to_string(), "ImportError: cannot import name setting");
py.run(
c"
for name in ('litellm.rust_bridge.settings', 'litellm.rust_bridge', 'litellm'):
for name in ('litellm.rust_bridge.host.settings', 'litellm.rust_bridge.host', 'litellm.rust_bridge', 'litellm'):
sys.modules.pop(name, None)
sys.modules.update(previous_modules)
",

2
litellm-rust/crates/python-bridge/src/routes/messages/host.rs
View file

@ -214,7 +214,7 @@ impl MessagesPythonHost {
) -> PyResult<MessagesShaping> {
let module = py.import(ROUTE_HOST_MODULE)?;
let capabilities: MessagesModelCapabilities = from_py(
&py.import("litellm.rust_bridge.model_capabilities")?
&py.import("litellm.rust_bridge.host.model_capabilities")?
.getattr("anthropic_model_capabilities")?
.call1((model, custom_llm_provider))?,
)?;

4
litellm-rust/crates/python-bridge/src/routes/ocr/project.rs
View file

@ -368,11 +368,11 @@ mod tests {
c"
import sys
import types
timeouts = types.ModuleType('litellm.rust_bridge.timeouts')
timeouts = types.ModuleType('litellm.rust_bridge.host.timeouts')
timeouts.timeout_to_seconds = lambda timeout: None if timeout is None else float(timeout)
sys.modules.setdefault('litellm', types.ModuleType('litellm'))
sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bridge'))
sys.modules['litellm.rust_bridge.timeouts'] = timeouts
sys.modules['litellm.rust_bridge.host.timeouts'] = timeouts
",
);
}

31
litellm-rust/crates/python-bridge/src/secrets/callback.rs
View file

@ -14,8 +14,7 @@ use pyo3::{
use super::error::{external_error, read_error};
const HANDLER_MODULE: &str = "litellm.secret_managers.secret_manager_handler";
const ENVIRONMENT_FALLBACK_LOG: &str =
"Defaulting to os.environ value for key=%s. An exception occurred - %s.\n\n%s";
const DIAGNOSTICS_MODULE: &str = "litellm.rust_bridge.host.diagnostics";
/// A secret manager whose reads execute in Python: a custom manager, a legacy compatible
/// client, or a manually assigned SDK client.
@ -112,16 +111,9 @@ impl ExternalSecretManager for PythonSecretManager {
}
fn log_environment_fallback(py: Python<'_>, name: &str, error: &PyErr) -> PyResult<()> {
let traceback = py
.import("traceback")?
.call_method1("format_exception", (error.value(py),))?;
let traceback = "".into_pyobject(py)?.call_method1("join", (traceback,))?;
py.import("litellm._logging")?
.getattr("verbose_logger")?
.call_method1(
"error",
(ENVIRONMENT_FALLBACK_LOG, name, error.value(py), traceback),
)?;
py.import(DIAGNOSTICS_MODULE)?
.getattr("log_environment_fallback")?
.call1((name, error.value(py)))?;
Ok(())
}
@ -139,7 +131,7 @@ mod tests {
use litellm_host_python::PythonContext;
use super::{HANDLER_MODULE, PythonSecretManager, python_name};
use super::{DIAGNOSTICS_MODULE, HANDLER_MODULE, PythonSecretManager, python_name};
use crate::secrets::python_error;
/// `sys.modules` is interpreter-global, so tests that install or rely on the handler module
@ -250,13 +242,24 @@ class Logger:
self.calls.append(args)
logging = types.ModuleType('litellm._logging')
logging.verbose_logger = Logger()
sys.modules.setdefault('litellm', types.ModuleType('litellm'))
for module in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host'):
sys.modules.setdefault(module, types.ModuleType(module))
sys.modules.setdefault('litellm._logging', logging)
",
None,
None,
)
.unwrap();
pyo3::types::PyModule::from_code(
py,
&std::ffi::CString::new(include_str!(
"../../../../../litellm/rust_bridge/host/diagnostics.py"
))
.unwrap(),
c"diagnostics.py",
&std::ffi::CString::new(DIAGNOSTICS_MODULE).unwrap(),
)
.unwrap();
py.import("litellm._logging")
.unwrap()
.getattr("verbose_logger")

8
litellm-rust/crates/python-bridge/src/secrets/python.rs
View file

@ -7,7 +7,9 @@ use pyo3::prelude::*;
use super::error::external_error;
/// Reads each secret through Python's `get_secret_str`, so the configured manager, the key
const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager";
/// Reads each secret through the host module's `read_secret_str`, so the configured manager, the key
/// management settings and the environment fallback behave exactly as they do in Python.
pub(super) struct PythonSecrets {
get_secret_str: Arc<Py<PyAny>>,
@ -17,8 +19,8 @@ pub(super) struct PythonSecrets {
impl PythonSecrets {
pub(super) fn new(py: Python<'_>) -> PyResult<Self> {
Ok(Self::reading_with(
py.import("litellm.secret_managers.main")?
.getattr("get_secret_str")?
py.import(SECRET_MANAGER_MODULE)?
.getattr("read_secret_str")?
.unbind(),
PythonContext::capture(py)?,
))

4
litellm-rust/crates/python-bridge/src/secrets/runtime.rs
View file

@ -15,6 +15,8 @@ use pyo3::{
use crate::execution::{run_async_value, run_sync_value};
const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager";
#[derive(Clone, PartialEq)]
struct Configuration {
system: KeyManagementSystem,
@ -101,7 +103,7 @@ impl NativeSecretManager {
return Ok(Some(native));
}
let config = py
.import("litellm.rust_bridge.secret_manager")?
.import(SECRET_MANAGER_MODULE)?
.getattr("native_secret_manager_config")?
.call1((client,))?;
if config.is_none() {

10
litellm-rust/crates/python-bridge/src/secrets/vault.rs
View file

@ -7,6 +7,8 @@ use pyo3::prelude::*;
use super::mutation::{error_value, http_message, json_value};
const HTTP_TRANSPORT_MODULE: &str = "litellm.rust_bridge.host.http_transport";
pub(super) fn failure_value(
py: Python<'_>,
failure: Failure,
@ -82,7 +84,7 @@ fn cause_message(py: Python<'_>, kind: FailureKind, context: &ErrorContext) -> P
kwargs.set_item("model", "default-model-name")?;
kwargs.set_item("llm_provider", "litellm-httpx-handler")?;
kwargs.set_item("headers", pyo3::types::PyDict::new(py))?;
py.import("litellm")?
py.import("litellm.exceptions")?
.getattr("Timeout")?
.call((), Some(&kwargs))?
.str()?
@ -173,9 +175,9 @@ impl ErrorContext {
Ok(Self {
timeout: timeout.map(|value| value.str()?.extract()).transpose()?,
aiohttp: py
.import("litellm.llms.custom_httpx.http_handler")?
.getattr("AsyncHTTPHandler")?
.call_method0("_should_use_aiohttp_transport")?
.import(HTTP_TRANSPORT_MODULE)?
.getattr("uses_aiohttp_transport")?
.call0()?
.extract()?,
})
}

11
litellm/__init__.py
View file

@ -170,6 +170,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"focus",
"mavvrik",
"vantage",
"ternary",
"posthog",
"levo",
"compression_interception",
@ -477,6 +478,7 @@ max_end_user_budget_id: Optional[str] = None
# pass through unchanged.
validate_end_user_id_in_db: bool = False
block_requests_for_models_without_pricing: bool = False
personal_key_model_access_from_teams: bool = False
disable_end_user_cost_tracking: Optional[bool] = None
disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
@ -721,6 +723,7 @@ aws_polly_models: Set = set()
transcribe_models: Set = set()
gigachat_models: Set = set()
llamagate_models: Set = set()
coralbricks_models: Set[str] = set()
reducto_models: Set = set()
bedrock_mantle_models: Set = set()
@ -1012,6 +1015,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
gigachat_models.add(key)
elif value.get("litellm_provider") == "llamagate":
llamagate_models.add(key)
elif value.get("litellm_provider") == "coralbricks":
coralbricks_models.add(key)
elif value.get("litellm_provider") == "reducto":
reducto_models.add(key)
elif value.get("litellm_provider") == "bedrock_mantle":
@ -1146,6 +1151,7 @@ model_list = list(
| docker_model_runner_models
| reducto_models
| bedrock_mantle_models
| coralbricks_models
| set(clarifai_models)
)
@ -1265,6 +1271,7 @@ def _build_models_by_provider() -> dict:
"transcribe": transcribe_models,
"gigachat": gigachat_models,
"llamagate": llamagate_models,
"coralbricks": coralbricks_models,
"reducto": reducto_models,
"bedrock_mantle": bedrock_mantle_models,
}
@ -1689,6 +1696,9 @@ if TYPE_CHECKING:
from .llms.openrouter.decisions.transformation import (
OpenRouterDecisionsConfig as OpenRouterDecisionsConfig,
)
from .llms.fireworks_ai.decisions.transformation import (
FireworksAIDecisionsConfig as FireworksAIDecisionsConfig,
)
from .llms.cloudflare.decisions.transformation import (
CloudflareDecisionsConfig as CloudflareDecisionsConfig,
)
@ -2277,6 +2287,7 @@ if TYPE_CHECKING:
supports_parallel_function_calling: Callable[..., bool]
supports_vision: Callable[..., bool]
supports_audio_input: Callable[..., bool]
supports_video_input: Callable[..., bool]
supports_audio_output: Callable[..., bool]
supports_system_messages: Callable[..., bool]
supports_reasoning: Callable[..., bool]

4
litellm/_lazy_imports_registry.py
View file

@ -36,6 +36,7 @@ UTILS_NAMES: Final = (
"supports_parallel_function_calling",
"supports_vision",
"supports_audio_input",
"supports_video_input",
"supports_audio_output",
"supports_system_messages",
"supports_reasoning",
@ -160,6 +161,7 @@ LLM_CONFIG_NAMES: Final = (
"PerplexityDecisionsConfig",
"TypeSafeDecisionsConfig",
"OpenRouterDecisionsConfig",
"FireworksAIDecisionsConfig",
"CloudflareDecisionsConfig",
"StrandsDeciderDecisionsConfig",
"DatabricksDecisionsConfig",
@ -481,6 +483,7 @@ _UTILS_IMPORT_MAP: Final = {
),
"supports_vision": (".utils", "supports_vision"),
"supports_audio_input": (".utils", "supports_audio_input"),
"supports_video_input": (".utils", "supports_video_input"),
"supports_audio_output": (".utils", "supports_audio_output"),
"supports_system_messages": (".utils", "supports_system_messages"),
"supports_reasoning": (".utils", "supports_reasoning"),
@ -722,6 +725,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
"TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),
"CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"),
"StrandsDeciderDecisionsConfig": (
".llms.strands_decider.decisions.transformation",

42
litellm/a2a_protocol/card_resolver.py
View file

@ -21,6 +21,7 @@ AGENT_CARD_WELL_KNOWN_PATH: str = "/.well-known/agent-card.json"
PREV_AGENT_CARD_WELL_KNOWN_PATH: str = "/.well-known/agent.json"
FOUNDRY_AGENT_CARD_PATH: Final = "/agentCard/v1.0"
AGENT_CARD_PATH_PARAM: Final = "agent_card_path"
A2A_PROTOCOL_VERSION_PARAM: Final = "a2a_protocol_version"
try:
from a2a.client import A2ACardResolver as _A2ACardResolver
@ -76,22 +77,43 @@ _CANONICAL_PROTOCOL_BINDINGS: Final = MappingProxyType(
)
_LEGACY_PROTOCOL_VERSION: Final = "0.3"
_A2A_PROTOCOL_VERSION_ALIASES: Final = MappingProxyType(
{
"0.3": "0.3",
"0.3.0": "0.3",
"1.0": "1.0",
"1.0.0": "1.0",
}
)
def normalize_agent_card_interfaces(agent_card: "AgentCard") -> "AgentCard":
def resolve_a2a_protocol_version(value: object) -> str | None:
if value is None:
return None
if isinstance(value, bool):
verbose_logger.warning("Ignoring invalid %s value: %r", A2A_PROTOCOL_VERSION_PARAM, value)
return None
if not isinstance(value, (str, int, float)):
verbose_logger.warning("Ignoring invalid %s value: %r", A2A_PROTOCOL_VERSION_PARAM, value)
return None
protocol_version: Final = _A2A_PROTOCOL_VERSION_ALIASES.get(str(value).strip())
if protocol_version is None:
verbose_logger.warning("Ignoring invalid %s value: %r", A2A_PROTOCOL_VERSION_PARAM, value)
return protocol_version
def normalize_agent_card_interfaces(agent_card: "AgentCard", protocol_version: str | None = None) -> "AgentCard":
"""
Canonicalize the supported interfaces of spec-adjacent agent cards.
Some A2A servers (e.g. LangGraph Platform) serve agent cards with lowercase
bindings like "jsonrpc", but a2a-sdk's ClientFactory matches bindings
case-sensitively against its uppercase TransportProtocol constants and fails
with "no compatible transports found." for spec-adjacent casings.
The same servers also speak the A2A 0.3 JSON dialect ("kind"-discriminated
payloads) while declaring protocolVersion "1.0", which a2a-sdk's strict v1
proto parsing rejects. A mis-cased binding fingerprints such a server, so its
declared version is downgraded to 0.3 to route the SDK's ClientFactory onto
its v0.3 compat transport, which speaks that dialect.
with "no compatible transports found." for spec-adjacent casings. The casing
fingerprint stopped working with langgraph-api 0.15, so operators can pin
the version per agent via ``litellm_params.a2a_protocol_version``. An explicit
"1.0" opts out of the mis-cased downgrade.
"""
normalized: Final = type(agent_card)()
normalized.CopyFrom(agent_card)
@ -101,6 +123,10 @@ def normalize_agent_card_interfaces(agent_card: "AgentCard") -> "AgentCard":
continue
interface.protocol_binding = canonical
interface.protocol_version = _LEGACY_PROTOCOL_VERSION
if protocol_version is not None:
for interface in normalized.supported_interfaces:
if interface.protocol_binding in _CANONICAL_PROTOCOL_BINDINGS.values():
interface.protocol_version = protocol_version
return normalized

8
litellm/a2a_protocol/exception_mapping_utils.py
View file

@ -23,6 +23,12 @@ if TYPE_CHECKING:
from a2a.client import Client as A2AClientType
_A2A_KIND_DIALECT_HINT: Final = (
" (the agent replied in the A2A 0.3 dialect while its card declares 1.0; "
'set litellm_params.a2a_protocol_version: "0.3" on this agent)'
)
try:
from a2a.client import Client, ClientConfig, create_client
@ -152,7 +158,7 @@ def map_a2a_exception(
# Default: wrap in generic A2AError
raise A2AError(
message=error_str,
message=error_str + _A2A_KIND_DIALECT_HINT if 'has no field named "kind"' in error_str else error_str,
model=model,
)

3
litellm/a2a_protocol/litellm_completion_bridge/handler.py
View file

@ -15,7 +15,7 @@ from typing import Any, Final
import litellm
from litellm._logging import verbose_logger
from litellm.a2a_protocol.card_resolver import AGENT_CARD_PATH_PARAM
from litellm.a2a_protocol.card_resolver import A2A_PROTOCOL_VERSION_PARAM, AGENT_CARD_PATH_PARAM
from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
A2ACompletionBridgeTransformation,
A2AStreamingContext,
@ -38,6 +38,7 @@ _AGENT_ONLY_PARAMS: Final = frozenset(
"agent_id",
"agent_card_params",
AGENT_CARD_PATH_PARAM,
A2A_PROTOCOL_VERSION_PARAM,
A2A_USER_API_KEY_HASH_PARAM,
}
)

15
litellm/a2a_protocol/main.py
View file

@ -72,10 +72,12 @@ except ImportError:
# Import our custom card resolver that supports multiple well-known paths
from litellm.a2a_protocol.card_resolver import (
A2A_PROTOCOL_VERSION_PARAM,
AGENT_CARD_PATH_PARAM,
LiteLLMA2ACardResolver,
get_agent_card_url,
normalize_agent_card_interfaces,
resolve_a2a_protocol_version,
)
from litellm.a2a_protocol.exception_mapping_utils import (
handle_a2a_localhost_retry,
@ -153,6 +155,10 @@ def _agent_card_path(litellm_params: Mapping[str, object]) -> str | None:
return configured_path if isinstance(configured_path, str) and configured_path else None
def _agent_protocol_version(litellm_params: Mapping[str, object]) -> str | None:
return resolve_a2a_protocol_version(litellm_params.get(A2A_PROTOCOL_VERSION_PARAM))
def _set_litellm_params_on_logging_obj(
kwargs: Mapping[str, object],
litellm_params: Mapping[str, object],
@ -498,6 +504,7 @@ async def asend_message(
base_url=api_base,
extra_headers=extra_headers,
relative_card_path=_agent_card_path(litellm_params),
protocol_version=_agent_protocol_version(litellm_params),
)
# Type assertion: a2a_client is guaranteed to be non-None here
@ -634,7 +641,7 @@ async def asend_message_streaming(
proxy_server_request: dict[str, object] | None = None,
agent_extra_headers: dict[str, str] | None = None,
**kwargs: object,
) -> AsyncIterator[Any]:
) -> AsyncIterator["SendStreamingMessageResponse | Mapping[str, object]"]:
"""
Async: Send a streaming message to an A2A agent.
@ -723,6 +730,7 @@ async def asend_message_streaming(
extra_headers=extra_headers,
streaming=True,
relative_card_path=_agent_card_path(litellm_params),
protocol_version=_agent_protocol_version(litellm_params),
)
assert a2a_client is not None
@ -770,6 +778,7 @@ async def create_a2a_client(
extra_headers: dict[str, str] | None = None,
streaming: bool = False,
relative_card_path: str | None = None,
protocol_version: str | None = None,
) -> "A2AClientType":
"""
Create an A2A client for the given agent URL.
@ -783,6 +792,7 @@ async def create_a2a_client(
extra_headers: Optional additional headers to include in requests
relative_card_path: Optional card path relative to ``base_url`` (e.g. ``agentCard/v1.0`` for a
Microsoft Foundry agent); when None the well-known paths are probed in order
protocol_version: Optional per-agent protocol version override for supported interfaces
Returns:
An initialized a2a.client.A2AClient instance
@ -819,7 +829,8 @@ async def create_a2a_client(
await resolver.get_agent_card(
relative_card_path=relative_card_path,
http_kwargs=_card_http_kwargs(extra_headers),
)
),
protocol_version=protocol_version,
)
a2a_client: Final = await create_client( # pyright: ignore[reportOptionalCall]

7
litellm/caching/caching.py
View file

@ -445,7 +445,7 @@ class Cache:
1. If caching groups are set, then return the caching group as the model https://docs.litellm.ai/docs/routing#caching-across-model-groups
2. Else if a model_group is set, then return the model_group as the model. This is used for all requests sent through the litellm.Router()
3. Else use the `model` passed in kwargs
3. Else use the `model` passed in kwargs, plus `model_id` when set (e.g. a SageMaker inference component)
"""
litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {}
metadata_sources: Final[tuple[dict, ...]] = (
@ -460,7 +460,10 @@ class Cache:
caching_group: Final = next(
(group for source in metadata_sources if (group := self._get_caching_group(source, model_group))), None
)
return caching_group or model_group or kwargs["model"]
model_id_suffix: Final = (
f" model_id: {kwargs['model_id']}" if "model_id" in kwargs and kwargs["model_id"] is not None else ""
)
return caching_group or model_group or f"{kwargs['model']}{model_id_suffix}"
def _get_caching_group(self, metadata: dict, model_group: str | None) -> str | None:
caching_groups: Final[list | None] = metadata.get("caching_groups", [])

110
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -74,7 +74,7 @@ if TYPE_CHECKING:
ChatCompletionToolReferenceObject,
OpenAIMessageContentListBlock,
)
from litellm.types.utils import Choices
from litellm.types.utils import Choices, Message
_CHAT_COMPLETION_FIELDS: Final = frozenset((*ModelResponse.model_fields, "usage"))
@ -312,6 +312,67 @@ def _as_chat_reasoning_items(
return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation:
shifted: Final = {
key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value
for key, value in annotation.items()
}
return cast(ChatCompletionAnnotation, shifted)
def _message_annotations(message: "Message") -> Sequence[ChatCompletionAnnotation]:
annotations: Sequence[ChatCompletionAnnotation] | None = getattr(message, "annotations", None)
return annotations or ()
def _message_reasoning_content(message: "Message") -> str | None:
reasoning_content: str | None = getattr(message, "reasoning_content", None)
return reasoning_content
def _message_reasoning_items(message: "Message") -> Sequence[ChatCompletionReasoningItem]:
reasoning_items: Sequence[ChatCompletionReasoningItem] | None = getattr(message, "reasoning_items", None)
return reasoning_items or ()
def _shifted_annotations(messages: Sequence["Message"], offsets: Sequence[int]) -> Iterator[ChatCompletionAnnotation]:
for message, offset in zip(messages, offsets):
for annotation in _message_annotations(message):
yield _shift_annotation(annotation, offset)
def _fold_choices_into_one(
choices: Sequence["Choices"],
tool_calls: Sequence[Mapping[str, object]],
trailing_reasoning_content: str | None,
trailing_reasoning_item: _BuiltReasoningItem | None,
) -> "Choices":
"""Clients that only read choices[0] must still see every text and tool call."""
from litellm.types.utils import Choices, Message
messages: Final = tuple(choice.message for choice in choices)
texts: Final = tuple(str(message.content or "") for message in messages)
offsets: Final = (0, *accumulate(len(text) for text in texts))
annotations: Final = tuple(_shifted_annotations(messages, offsets))
reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
reasoning_items: Final = (
*chain.from_iterable(_message_reasoning_items(m) for m in messages),
*(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
)
return Choices(
message=Message(
role=messages[0].role if messages else "assistant",
content="".join(texts) if messages else None,
annotations=list(annotations) or None,
tool_calls=list(tool_calls) or None,
reasoning_content=" ".join(value for value in reasoning_contents if value) or None,
reasoning_items=_as_chat_reasoning_items(reasoning_items),
),
finish_reason="tool_calls" if tool_calls else "stop",
index=0,
)
_ToolChoiceT = TypeVar("_ToolChoiceT")
@ -958,47 +1019,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
else:
pass # don't fail request if item in list is not supported
if accumulated_tool_calls and choices:
last_choice: Final = choices[-1]
last_reasoning_content: Final = getattr(last_choice.message, "reasoning_content", None)
last_reasoning_items: Final = getattr(last_choice.message, "reasoning_items", None)
merged_reasoning_content: Final = (
" ".join(value for value in (last_reasoning_content, reasoning_content) if value) or None
)
merged_reasoning_items: Final = _as_chat_reasoning_items(
(
*(last_reasoning_items or ()),
*(() if pending_reasoning_item is None else (pending_reasoning_item,)),
)
)
merged_message: Final = Message(
role=last_choice.message.role,
content=last_choice.message.content,
annotations=getattr(last_choice.message, "annotations", None),
if not choices and not accumulated_tool_calls:
return choices
return [
_fold_choices_into_one(
choices=choices,
tool_calls=accumulated_tool_calls,
reasoning_content=merged_reasoning_content,
reasoning_items=merged_reasoning_items,
trailing_reasoning_content=reasoning_content,
trailing_reasoning_item=pending_reasoning_item,
)
return [
*choices[:-1],
Choices(message=merged_message, finish_reason="tool_calls", index=last_choice.index),
]
if accumulated_tool_calls:
msg = Message(
content=None,
tool_calls=accumulated_tool_calls,
reasoning_content=reasoning_content,
reasoning_items=cast(
list[ChatCompletionReasoningItem] | None,
([pending_reasoning_item] if pending_reasoning_item is not None else None),
),
)
choices.append(Choices(message=msg, finish_reason="tool_calls", index=index))
reasoning_content = None
pending_reasoning_item = None
return choices
]
@staticmethod
def _build_empty_incomplete_choice(

Some files were not shown because too many files have changed in this diff Show more