scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 154

Merge pull request #25521 from BerriAI/main

Browse source 
merge main
This commit is contained in:
Sameer Kankute 2026-04-11 00:30:03 +05:30 • committed by GitHub
commit 5805609fb6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
965 changed files with 34246 additions and 13317 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

413
.circleci/config.yml
View file

@ -1330,6 +1330,57 @@ jobs:
paths:
- audio_coverage.xml
- audio_coverage
redis_caching_unit_tests:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip uv
uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-xdist==3.6.1"
pip install "pytest-rerunfailures==14.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv \
tests/local_testing/test_dual_cache.py \
tests/local_testing/test_redis_batch_optimizations.py \
tests/local_testing/test_router_utils.py \
--cov=litellm --cov-report=xml \
-x -s -v --junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1
no_output_timeout: 20m
- run:
name: Rename the coverage files
command: |
mv coverage.xml redis_caching_coverage.xml
mv .coverage redis_caching_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- redis_caching_coverage.xml
- redis_caching_coverage
installing_litellm_on_python:
docker:
- image: cimg/python:3.11
@ -2868,114 +2919,6 @@ jobs:
- store_test_results:
path: test-results
proxy_e2e_azure_batches_tests:
machine:
image: ubuntu-2204:2023.10.1
resource_class: large
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Docker CLI
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.12
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.12 -y
conda activate myenv
python --version
- run:
name: Install Poetry
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
pip install poetry
- run:
name: Install dockerize
command: |
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=llmproxy \
-e POSTGRES_PASSWORD=dbpassword9090 \
-e POSTGRES_DB=litellm \
-p 5432:5432 \
postgres:15
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Install system dependencies
command: |
sudo apt-get update -y
sudo apt-get install -y libpq-dev
- run:
name: Install Dependencies
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry config virtualenvs.in-project true
poetry install --with dev,proxy-dev --extras "proxy"
poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
- run:
name: Setup litellm-enterprise
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry run pip install --force-reinstall --no-deps -e enterprise/
- run:
name: Generate Prisma client
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry run prisma generate --schema litellm/proxy/schema.prisma
- run:
name: Run Prisma migrations
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
cd litellm/proxy
poetry run prisma migrate deploy --schema schema.prisma
cd ../..
- run:
name: Run Azure Batch E2E Tests
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
export USE_LOCAL_LITELLM=true
export USE_MOCK_MODELS=true
export USE_STATE_TRACKER=true
export LITELLM_LOG=DEBUG
poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
-vv -s -k "test_e2e_managed_batch" \
--tb=short \
--maxfail=3 \
--durations=10 \
--junitxml=test-results/junit.xml
no_output_timeout: 15m
upload-coverage:
docker:
- image: cimg/python:3.9
@ -2997,7 +2940,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage
coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@ -3182,6 +3125,117 @@ jobs:
CI=true npm run test -- --run \
--pool forks --poolOptions.forks.maxForks=8
e2e_ui_testing:
docker:
- image: cimg/python:3.12-browsers
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
- image: cimg/postgres:16.0
environment:
POSTGRES_USER: e2euser
POSTGRES_PASSWORD: e2epassword
POSTGRES_DB: litellm_e2e
resource_class: large
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
CI: "true"
steps:
- checkout
- setup_google_dns
- restore_cache:
keys:
- ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }}
- run:
name: Install Python dependencies
command: |
python -m pip install --upgrade pip uv
uv pip install --system -r requirements.txt
pip install "prisma==0.11.0"
prisma generate --schema litellm/proxy/schema.prisma
- save_cache:
key: ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }}
paths:
- ~/.local/lib
- ~/.local/bin
- restore_cache:
keys:
- ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- run:
name: Install Node dependencies and Playwright
command: |
cd ui/litellm-dashboard
npm ci
npx playwright install chromium --with-deps
- save_cache:
key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
paths:
- ui/litellm-dashboard/node_modules
- run:
name: Build UI from source
command: |
cd ui/litellm-dashboard
npm run build
cp -r out/ ../../litellm/proxy/_experimental/out/
# Restructure HTML so extensionless routes work (login.html -> login/index.html)
find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
done
- run:
name: Wait for PostgreSQL
command: dockerize -wait tcp://localhost:5432 -timeout 30s
- run:
name: Push Prisma schema
command: prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Seed database
command: |
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
-f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
- run:
name: Start mock LLM server
command: python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
background: true
- run:
name: Start LiteLLM proxy
environment:
LITELLM_MASTER_KEY: "sk-1234"
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
DISABLE_SCHEMA_UPDATE: "true"
SERVER_ROOT_PATH: ""
PROXY_LOGOUT_URL: ""
command: |
python -m litellm.proxy.proxy_cli \
--config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
--port 4000
background: true
- run:
name: Wait for proxy to be ready
command: |
for i in $(seq 1 60); do
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true)
if [ "$HTTP_CODE" = "200" ]; then
echo "Proxy is ready"
exit 0
fi
sleep 2
done
echo "Proxy failed to start"
exit 1
- run:
name: Run Playwright E2E tests
command: |
cd ui/litellm-dashboard
npx playwright test --config e2e_tests/playwright.config.ts
no_output_timeout: 10m
- store_artifacts:
path: ui/litellm-dashboard/test-results
destination: e2e-test-results
- store_artifacts:
path: ui/litellm-dashboard/playwright-report
destination: e2e-playwright-report
build_docker_database_image:
machine:
image: ubuntu-2204:2024.04.1
@ -3207,102 +3261,6 @@ jobs:
paths:
- litellm-docker-database.tar.zst
e2e_ui_testing:
machine:
image: ubuntu-2204:2023.10.1
resource_class: large
working_directory: ~/project
parameters:
browser:
type: string
steps:
- checkout
- setup_google_dns
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Install Dependencies
command: |
npm install -D @playwright/test
- run:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Install Neon CLI
command: |
npm i -g neonctl
- run:
name: Create Neon branch
command: |
export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ")
echo "Expires at: $EXPIRES_AT"
neon branches create \
--project-id $NEON_PROJECT_ID \
--name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--expires-at $EXPIRES_AT \
--parent br-fancy-paper-ad1olsb3 \
--api-key $NEON_API_KEY || true
- run:
name: Run Docker container
command: |
E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--database-name yuneng-trial-db \
--role neondb_owner)
echo $E2E_UI_TEST_DATABASE_URL
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e UI_USERNAME="admin" \
-e UI_PASSWORD="gm" \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
--name litellm-docker-database-<< parameters.browser >> \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
- run:
name: Install curl and dockerize
command: |
sudo apt-get update
sudo apt-get install -y curl
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f litellm-docker-database-<< parameters.browser >>
background: true
- run:
name: Wait for app to be ready
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Run Playwright Tests
command: |
npx playwright test \
--project << parameters.browser >> \
--config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
--reporter=html \
--output=test-results
no_output_timeout: 15m
- store_artifacts:
path: test-results
destination: playwright-results
- store_artifacts:
path: playwright-report
destination: playwright-report
prisma_schema_sync:
machine:
@ -3531,32 +3489,12 @@ workflows:
only:
- main
- /litellm_.*/
# - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli
# name: e2e_ui_testing_chromium
# browser: chromium
# context: e2e_ui_tests
# requires:
# - ui_build
# - build_docker_database_image
# - prisma_schema_sync
# filters:
# branches:
# only:
# - main
# - /litellm_.*/
# - e2e_ui_testing:
# name: e2e_ui_testing_firefox
# browser: firefox
# context: e2e_ui_tests
# requires:
# - ui_build
# - build_docker_database_image
# - prisma_schema_sync
# filters:
# branches:
# only:
# - main
# - /litellm_.*/
- e2e_ui_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- build_and_test:
requires:
- build_docker_database_image
@ -3627,12 +3565,6 @@ workflows:
only:
- main
- /litellm_.*/
- proxy_e2e_azure_batches_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- llm_translation_testing:
filters:
branches:
@ -3751,6 +3683,12 @@ workflows:
only:
- main
- /litellm_.*/
- redis_caching_unit_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- upload-coverage:
requires:
- realtime_translation_testing
@ -3769,6 +3707,7 @@ workflows:
- image_gen_testing
- logging_testing
- audio_testing
- redis_caching_unit_tests
- langfuse_logging_unit_tests
- local_testing_part1
- local_testing_part2

48
.github/workflows/_test-unit-base.yml vendored
View file

@ -27,6 +27,10 @@ on:
required: false
type: number
default: 10
artifact-name:
description: "Unique name for the coverage artifact (must be unique per run)"
required: true
type: string
permissions:
contents: read
@ -93,4 +97,46 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--dist=loadscope \
--durations=20
--durations=20 \
--cov=litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
- name: Save coverage report
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage.xml
retention-days: 1
upload-coverage:
name: Upload coverage to Codecov
needs: run
if: always()
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
pull-requests: write
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Download coverage report
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage-reports
merge-multiple: true
- name: Upload to Codecov
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
fail_ci_if_error: false

71
.github/workflows/_test-unit-services-base.yml vendored
View file

@ -27,23 +27,17 @@ on:
required: false
type: number
default: 10
enable-redis:
description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars"
required: false
type: boolean
default: false
enable-postgres:
description: "Start a local Postgres service container and run Prisma migrations"
required: false
type: boolean
default: false
artifact-name:
description: "Unique name for the coverage artifact (must be unique per run)"
required: false
type: string
default: "run"
secrets:
REDIS_HOST:
required: false
REDIS_PORT:
required: false
REDIS_PASSWORD:
required: false
DATABASE_URL:
required: false
POSTGRES_USER:
@ -61,11 +55,8 @@ jobs:
timeout-minutes: ${{ inputs.timeout-minutes }}
# Environment is derived from the enable-* flags, not caller-controllable.
# This prevents callers from passing arbitrary environment names to bypass secret scoping.
# Note: Postgres service container always starts (GHA limitation), so any Redis job
# also needs Postgres secrets → uses integration-redis-postgres, not integration-redis.
environment: >-
${{
inputs.enable-redis && 'integration-redis-postgres' ||
inputs.enable-postgres && 'integration-postgres' ||
''
}}
@ -141,9 +132,6 @@ jobs:
WORKERS: ${{ inputs.workers }}
RERUNS: ${{ inputs.reruns }}
DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }}
REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }}
REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }}
REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }}
run: |
if [ "${WORKERS}" = "0" ]; then
poetry run pytest ${TEST_PATH:?} \
@ -151,7 +139,10 @@ jobs:
--maxfail="${MAX_FAILURES}" \
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20
--durations=20 \
--cov=litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
poetry run pytest ${TEST_PATH:?} \
--tb=short -vv \
@ -160,5 +151,47 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--dist=loadscope \
--durations=20
--durations=20 \
--cov=litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi
- name: Save coverage report
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage.xml
retention-days: 1
upload-coverage:
name: Upload coverage to Codecov
needs: run
if: always()
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
pull-requests: write
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Download coverage report
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage-reports
merge-multiple: true
- name: Upload to Codecov
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
fail_ci_if_error: false

16
.github/workflows/create-release.yml vendored
View file

@ -48,7 +48,21 @@ jobs:
const cosignSection = [
`## Verify Docker Image Signature`,
``,
`All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`,
`All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit \`0112e53\`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).`,
``,
`**Verify using the pinned commit hash (recommended):**`,
``,
`A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:`,
``,
'```bash',
`cosign verify \\`,
` --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \\`,
` ghcr.io/berriai/litellm:${tag}`,
'```',
``,
`**Verify using the release tag (convenience):**`,
``,
`Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:`,
``,
'```bash',
`cosign verify \\`,

0
.github/workflows/run_llm_translation_tests.py vendored Executable file → Normal file
View file

2
.github/workflows/scorecard.yml vendored
View file

@ -42,6 +42,6 @@ jobs:
retention-days: 5
- name: Upload to code scanning
uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1
uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
with:
sarif_file: results.sarif

214
.github/workflows/test-litellm-matrix.yml vendored
View file

@ -1,214 +0,0 @@
name: LiteLLM Unit Tests (Matrix)
on:
pull_request:
branches: [main]
permissions:
contents: read
# Cancel in-progress runs for the same PR
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 20 # Increased from 15 to 20
strategy:
fail-fast: false
matrix:
test-group:
# tests/test_litellm split by subdirectory (~560 files total)
# Vertex AI tests separated for better isolation (prevent auth/env pollution)
- name: "llms-vertex"
path: "tests/test_litellm/llms/vertex_ai"
workers: 1
reruns: 2
- name: "llms-other"
path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
workers: 2
reruns: 2
# tests/test_litellm/proxy split by subdirectory (~180 files total)
- name: "proxy-guardrails"
path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
workers: 2
reruns: 2
- name: "proxy-core"
path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
workers: 2
reruns: 2
- name: "proxy-misc"
path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
workers: 2
reruns: 2
- name: "integrations"
path: "tests/test_litellm/integrations"
workers: 2
reruns: 3 # Integration tests tend to be flakier
- name: "core-utils"
path: "tests/test_litellm/litellm_core_utils"
workers: 2
reruns: 1
- name: "other-1"
# responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
workers: 2
reruns: 2
- name: "other-2"
# enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
workers: 2
reruns: 2
- name: "other-3"
# remaining dirs ≈ 8.0k lines
path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
workers: 2
reruns: 2
- name: "root"
path: "tests/test_litellm/test_*.py"
workers: 2
reruns: 2
# tests/proxy_unit_tests split alphabetically (~48 files total)
- name: "proxy-unit-a1"
# test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
path: "tests/proxy_unit_tests/test_[a-j]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-a2"
# test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
path: "tests/proxy_unit_tests/test_[k-o]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b1"
# lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b2"
# proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
path: "tests/proxy_unit_tests/test_proxy_server.py"
workers: 2
reruns: 1
- name: "proxy-unit-b3"
# proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
workers: 2
reruns: 1
- name: "proxy-unit-b4"
# proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
path: "tests/proxy_unit_tests/test_proxy_utils.py"
workers: 2
reruns: 1
- name: "proxy-unit-b5"
# proxy_token_counter (1279) - runs independently from utils
path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
workers: 2
reruns: 1
- name: "proxy-unit-b6"
# test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
path: "tests/proxy_unit_tests/test_[r-t]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b7"
# test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
path: "tests/proxy_unit_tests/test_[u-z]*.py"
workers: 2
reruns: 1
name: test (${{ matrix.test-group.name }})
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install Poetry
run: pip install 'poetry==2.3.2'
- name: Cache Poetry dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
with:
path: |
~/.cache/pypoetry
~/.cache/pip
.venv
key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
restore-keys: |
${{ runner.os }}-poetry-
- name: Install dependencies
run: |
poetry config virtualenvs.in-project true
poetry install --with dev,proxy-dev --extras "proxy semantic-router"
# pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
poetry run pip install google-genai==1.22.0 \
google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
- name: Setup litellm-enterprise
run: |
poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Generate Prisma client
env:
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
run: |
poetry run pip install nodejs-wheel-binaries==24.13.1
poetry run prisma generate --schema litellm/proxy/schema.prisma
- name: Run tests - ${{ matrix.test-group.name }}
run: |
poetry run pytest ${{ matrix.test-group.path }} \
--tb=short -vv \
--maxfail=10 \
-n ${{ matrix.test-group.workers }} \
--reruns ${{ matrix.test-group.reruns }} \
--reruns-delay 1 \
--dist=loadscope \
--durations=20 \
--cov=litellm \
--cov-report=xml:coverage-${{ matrix.test-group.name }}.xml \
--cov-config=pyproject.toml
- name: Save coverage report
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: coverage-${{ matrix.test-group.name }}
path: coverage-${{ matrix.test-group.name }}.xml
retention-days: 1
upload-coverage:
name: Upload coverage to Codecov
needs: test
if: always()
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write # Required for OIDC tokenless upload
pull-requests: write # Required for Codecov PR comments
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Download all coverage reports
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: coverage-*
path: coverage-reports
merge-multiple: true
- name: Upload to Codecov
uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
fail_ci_if_error: false

97
.github/workflows/test-proxy-e2e-azure-batches.yml vendored
View file

@ -1,97 +0,0 @@
name: Proxy E2E Azure Batches Tests
on:
pull_request:
branches: [main]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
proxy_e2e_azure_batches_tests:
runs-on: ubuntu-latest
timeout-minutes: 30
services:
postgres:
image: postgres:15
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install Poetry
run: pip install 'poetry==2.3.2'
- name: Cache Poetry dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
with:
path: |
~/.cache/pypoetry
~/.cache/pip
.venv
key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }}
restore-keys: |
${{ runner.os }}-poetry-e2e-batches-
${{ runner.os }}-poetry-
- name: Install dependencies
run: |
poetry config virtualenvs.in-project true
poetry install --with dev,proxy-dev --extras "proxy"
poetry run pip install psycopg2-binary==2.9.11 uvicorn==0.42.0 fastapi==0.135.2 httpx==0.28.1 tenacity==9.1.4
- name: Setup litellm-enterprise
run: |
poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Generate Prisma client
env:
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
run: |
poetry run pip install nodejs-wheel-binaries==24.13.1
poetry run prisma generate --schema litellm/proxy/schema.prisma
- name: Run Prisma migrations
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
run: |
cd litellm/proxy
poetry run prisma migrate deploy --schema schema.prisma
cd ../..
- name: Run Azure Batch E2E Tests
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
USE_LOCAL_LITELLM: "true"
USE_MOCK_MODELS: "true"
USE_STATE_TRACKER: "true"
LITELLM_LOG: DEBUG
run: |
poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
-vv -s -k "test_e2e_managed_batch" \
--tb=short \
--maxfail=3 \
--durations=10

38
.github/workflows/test-unit-caching-redis.yml vendored
View file

@ -1,38 +0,0 @@
name: "Unit Tests: Caching (Redis)"
# Uses cloud Redis credentials — only runs on trusted branches, not PRs.
# This prevents external PRs from accessing Redis credentials.
on:
push:
branches: [main, "litellm_*"]
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
caching-redis:
uses: ./.github/workflows/_test-unit-services-base.yml
with:
# Redis-only tests that do NOT require provider API keys.
# Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py,
# test_router_caching.py) are in Phase 3 integration workflows.
test-path: >-
tests/local_testing/test_dual_cache.py
tests/local_testing/test_redis_batch_optimizations.py
tests/local_testing/test_router_utils.py
workers: 2
reruns: 2
timeout-minutes: 20
enable-redis: true
enable-postgres: false
secrets:
REDIS_HOST: ${{ secrets.REDIS_HOST }}
REDIS_PORT: ${{ secrets.REDIS_PORT }}
REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
DATABASE_URL: ${{ secrets.DATABASE_URL }}
POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

3
.github/workflows/test-unit-core-utils.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -18,3 +20,4 @@ jobs:
test-path: "tests/test_litellm/litellm_core_utils"
workers: 2
reruns: 1
artifact-name: core-utils

3
.github/workflows/test-unit-enterprise-routing.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -22,3 +24,4 @@ jobs:
tests/test_litellm/router_strategy
workers: 2
reruns: 2
artifact-name: enterprise-routing

3
.github/workflows/test-unit-integrations.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -18,3 +20,4 @@ jobs:
test-path: "tests/test_litellm/integrations"
workers: 2
reruns: 3
artifact-name: integrations

10
.github/workflows/test-unit-llm-providers.yml vendored
View file

@ -14,16 +14,26 @@ concurrency:
jobs:
vertex-ai:
name: Vertex AI
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/llms/vertex_ai"
workers: 1
reruns: 2
artifact-name: llm-vertex-ai
other-providers:
name: All Other Providers
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
workers: 2
reruns: 2
artifact-name: llm-other-providers

3
.github/workflows/test-unit-misc.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -29,3 +31,4 @@ jobs:
tests/test_litellm/test_*.py
workers: 2
reruns: 2
artifact-name: misc

3
.github/workflows/test-unit-proxy-auth.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -18,3 +20,4 @@ jobs:
test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client"
workers: 2
reruns: 2
artifact-name: proxy-auth

6
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -14,6 +14,10 @@ concurrency:
jobs:
proxy-db:
permissions:
contents: read
id-token: write
pull-requests: write
strategy:
fail-fast: false
matrix:
@ -37,8 +41,8 @@ jobs:
workers: ${{ matrix.workers }}
reruns: 2
timeout-minutes: ${{ matrix.timeout }}
enable-redis: false
enable-postgres: true
artifact-name: proxy-db-${{ matrix.test-group }}
secrets:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
POSTGRES_USER: ${{ secrets.POSTGRES_USER }}

3
.github/workflows/test-unit-proxy-endpoints.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -33,3 +35,4 @@ jobs:
tests/test_litellm/proxy/ui_crud_endpoints
workers: 2
reruns: 2
artifact-name: proxy-endpoints

3
.github/workflows/test-unit-proxy-infra.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -26,3 +28,4 @@ jobs:
tests/test_litellm/proxy/test_*.py
workers: 2
reruns: 2
artifact-name: proxy-infra

3
.github/workflows/test-unit-responses-caching-types.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -18,3 +20,4 @@ jobs:
test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
workers: 2
reruns: 2
artifact-name: responses-caching-types

4
.github/workflows/test-unit-security.yml vendored
View file

@ -7,6 +7,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@ -20,8 +22,8 @@ jobs:
workers: 1
reruns: 2
timeout-minutes: 20
enable-redis: false
enable-postgres: true
artifact-name: security
secrets:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
POSTGRES_USER: ${{ secrets.POSTGRES_USER }}

40
.pre-commit-config.yaml
View file

@ -1,40 +0,0 @@
repos:
- repo: local
hooks:
- id: pyright
name: pyright
entry: pyright
language: system
types: [python]
files: ^(litellm/|litellm_proxy_extras/|enterprise/)
- id: isort
name: isort
entry: isort
language: system
types: [python]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
exclude: ^litellm/__init__.py$
- id: black
name: black
entry: poetry run black
language: system
types: [python]
files: (litellm/|litellm_proxy_extras/).*\.py
- repo: https://github.com/pycqa/flake8
rev: 7.0.0 # The version of flake8 to use
hooks:
- id: flake8
exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/
additional_dependencies: [flake8-print]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- repo: https://github.com/python-poetry/poetry
rev: 1.8.0
hooks:
- id: poetry-check
files: ^(pyproject.toml|litellm-proxy-extras/pyproject.toml)$
- repo: local
hooks:
- id: check-files-match
name: Check if files match
entry: python3 ci_cd/check_files_match.py
language: system

12
.trivyignore
View file

@ -1,12 +0,0 @@
# LiteLLM Trivy Ignore File
# CVEs listed here are temporarily allowlisted pending fixes
# Next.js vulnerabilities in UI dashboard (next@14.2.35)
# Allowlisted: 2026-01-31, 7-day fix timeline
# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+
# HIGH: DoS via request deserialization
GHSA-h25m-26qc-wcjf
# MEDIUM: Image Optimizer DoS
CVE-2025-59471

2
AGENTS.md
View file

@ -254,7 +254,7 @@ See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`.
- The `--timeout` pytest flag is NOT available; don't pass it.
- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
- Black `--check` may report pre-existing formatting issues; this does not block test runs.
- **Before committing, always run `poetry run black .` to format your code.** Black formatting is enforced in CI.
- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file.
### Lint

1
CLAUDE.md
View file

@ -20,6 +20,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
- `make format` - Apply Black code formatting
- `make lint-ruff` - Run Ruff linting only
- `make lint-mypy` - Run MyPy type checking only
- **Before committing, always run `poetry run black .` to format your code.** Black formatting is enforced in CI.
### Single Test Files
- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file

13
CONTRIBUTING.md
View file

@ -149,6 +149,19 @@ Apply formatting (auto-fixes issues):
make format
```
> **Black formatting is enforced in CI.** All PRs must pass the Black formatting check.
>
> - **AI coding agents** (Claude Code, Copilot, Cursor, etc.): `AGENTS.md` and `CLAUDE.md` instruct agents to run `poetry run black .` before committing.
> - **VS Code users**: Install the [Black Formatter extension](https://marketplace.visualstudio.com/items?itemName=ms-python.black-formatter) and enable format-on-save:
> ```json
> {
> "[python]": {
> "editor.defaultFormatter": "ms-python.black-formatter",
> "editor.formatOnSave": true
> }
> }
> ```
### CI Compatibility
To ensure your changes will pass CI, run the exact same checks locally:

39
README.md
View file

@ -2,20 +2,24 @@
🚅 LiteLLM
</h1>
<p align="center">
<p align="center">Call 100+ LLMs in OpenAI format. [Bedrock, Azure, OpenAI, VertexAI, Anthropic, Groq, etc.]
<p align="center">LiteLLM AI Gateway
</p>
<p align="center">Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.</p>
<p align="center">
<a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
<a href="https://railway.app/template/HLP0Ub?referralCode=jch2ME">
<img src="https://railway.app/button.svg" alt="Deploy on Railway">
<a href="https://railway.com/deploy/RhvhdC?referralCode=7mRv9K&utm_medium=integration&utm_source=template&utm_campaign=generic">
<img src="https://railway.com/button.svg" alt="Deploy on Railway">
</a>
</p>
</p>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://litellm.ai/enterprise"target="_blank">Enterprise Tier</a> | <a href="https://litellm.ai/" target="_blank">Website</a></h4>
<h4 align="center">
<a href="https://pypi.org/project/litellm/" target="_blank">
<img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
</a>
<a href="https://github.com/BerriAI/litellm" target="_blank">
<img src="https://img.shields.io/github/stars/BerriAI/litellm.svg?style=social" alt="GitHub Stars">
</a>
<a href="https://www.ycombinator.com/companies/berriai">
<img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23">
</a>
@ -400,9 +404,36 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
2. Install dependencies `npm install`
3. Run `npm run dev` to start the dashboard
# Verify Docker Image Signatures
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
**Verify using a release tag (convenience):**
Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
# Enterprise
For companies that need better security, user management and professional support
[Get an Enterprise License](https://litellm.ai/enterprise)
[Talk to founders](https://enterprise.litellm.ai/demo)
This covers:

36
ci_cd/.grype.yaml
View file

@ -1,36 +0,0 @@
ignore:
- vulnerability: CVE-2026-22184
reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
# Wolfi base image: Python 3.13 and Node from apk have no fixed builds in Wolfi yet / not applicable
- vulnerability: CVE-2025-55130
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-59465
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-55131
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-59466
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2026-21637
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-55132
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: GHSA-hx9q-6w63-j58v
reason: orjson dumps recursion; allowlisted
- vulnerability: GHSA-73rr-hh4g-fpgx
reason: diff npm transitive dep; override in package.json, allowlisted
- vulnerability: CVE-2026-0865
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15282
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2026-0672
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15366
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15367
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-11468
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-12781
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2026-1299
reason: Python 3.13 in Wolfi base; no fixed apk build yet

19
ci_cd/publish-proxy-extras.sh
View file

@ -1,19 +0,0 @@
#!/bin/bash
# Exit on error
set -e
echo "🚀 Building and publishing litellm-proxy-extras"
# Navigate to litellm-proxy-extras directory
cd "$(dirname "$0")/../litellm-proxy-extras"
# Build the package
echo "📦 Building package..."
poetry build
# Publish to PyPI
echo "🌎 Publishing to PyPI..."
poetry publish
echo "✅ Done! Package published successfully"

262
ci_cd/security_scans.sh
View file

@ -1,262 +0,0 @@
#!/bin/bash
# Security Scans Script for LiteLLM
# This script runs comprehensive security scans including Trivy and Grype
set -e
echo "Starting security scans for LiteLLM..."
# Function to install Trivy and required tools
install_trivy() {
echo "Installing Trivy and required tools..."
TRIVY_VERSION="0.35.0"
sudo apt-get update
sudo apt-get install -y wget jq curl bsdmainutils
wget -qO trivy.deb "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb"
sudo dpkg -i trivy.deb
rm trivy.deb
echo "Trivy ${TRIVY_VERSION} installed successfully"
}
# Function to install Grype
install_grype() {
echo "Installing Grype..."
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
echo "Grype installed successfully"
}
# Function to install ggshield
install_ggshield() {
echo "Installing ggshield..."
pip3 install --upgrade pip
pip3 install ggshield
echo "ggshield installed successfully"
}
# # Function to run secret detection scans
# run_secret_detection() {
# echo "Running secret detection scans..."
# if ! command -v ggshield &> /dev/null; then
# install_ggshield
# fi
# # Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
# if [ -z "$GITGUARDIAN_API_KEY" ]; then
# echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
# echo "ggshield requires a GitGuardian API key to scan for secrets."
# echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
# exit 1
# fi
# echo "Scanning codebase for secrets..."
# echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
# echo "ggshield will automatically handle rate limits and retry as needed."
# echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
# # Use --recursive for directory scanning and auto-confirm if prompted
# # .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
# # GITGUARDIAN_API_KEY environment variable will be used for authentication
# echo y | ggshield secret scan path . --recursive || {
# echo ""
# echo "=========================================="
# echo "ERROR: Secret Detection Failed"
# echo "=========================================="
# echo "ggshield has detected secrets in the codebase."
# echo "Please review discovered secrets above, revoke any actively used secrets"
# echo "from underlying systems and make changes to inject secrets dynamically at runtime."
# echo ""
# echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
# echo "=========================================="
# echo ""
# exit 1
# }
# echo "Secret detection scans completed successfully"
# }
# Function to run Trivy scans
run_trivy_scans() {
echo "Running Trivy scans..."
echo "Scanning LiteLLM Docs..."
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
echo "Scanning LiteLLM UI..."
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
echo "Trivy scans completed successfully"
}
# Function to build and scan Docker images with Grype
run_grype_scans() {
echo "Running Grype scans..."
# Temporarily add wheel files to .dockerignore for security scans
echo "Temporarily modifying .dockerignore to exclude problematic wheel files..."
cp .dockerignore .dockerignore.backup 2>/dev/null || touch .dockerignore.backup
echo "/*.whl" >> .dockerignore
# Build and scan Dockerfile.database
echo "Building and scanning Dockerfile.database..."
docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
grype litellm-database:latest --config ci_cd/.grype.yaml --fail-on critical
# Build and scan main Dockerfile
echo "Building and scanning main Dockerfile..."
docker build --no-cache -t litellm:latest .
grype litellm:latest --config ci_cd/.grype.yaml --fail-on critical
# Restore original .dockerignore
echo "Restoring original .dockerignore..."
mv .dockerignore.backup .dockerignore
# Scan the locally built LiteLLM image for vulnerabilities with CVSS >= 4.0
echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
echo "Using locally built image: litellm:latest"
# Allowlist of CVEs to be ignored in failure threshold/reporting
# - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
# - GHSA-4xh5-x5gv-qwph: GitHub Security Advisory alias for CVE-2025-8869
# - GHSA-5j98-mcp5-4vw2: glob CLI command injection via -c/--cmd; glob CLI is not used in the litellm runtime image,
# and the vulnerable versions are pulled in only via OS-level/node tooling outside of our application code
ALLOWED_CVES=(
"CVE-2025-8869"
"GHSA-4xh5-x5gv-qwph"
"CVE-2025-8291" # no fix available as of Oct 11, 2025
"GHSA-5j98-mcp5-4vw2"
"CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
"CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
"CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
"CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet
"CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
"CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
"CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
"CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
"GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
"GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
"GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
"CVE-2025-59465" # Node only used for Admin UI build/prisma
"CVE-2025-55131" # Node only used for Admin UI build/prisma
"CVE-2025-59466" # Node only used for Admin UI build/prisma
"CVE-2025-55130" # Node only used for Admin UI build/prisma
"CVE-2025-59467" # Node only used for Admin UI build/prisma
"CVE-2026-21637" # Node only used for Admin UI build/prisma
"CVE-2025-55132" # Node only used for Admin UI build/prisma
"GHSA-hx9q-6w63-j58v" # orjson dumps recursion; allowlisted
"CVE-2025-15281" # No fix available yet
"CVE-2026-0865" # No fix available yet
"CVE-2025-15282" # No fix available yet
"CVE-2026-0672" # No fix available yet
"CVE-2025-15366" # No fix available yet
"CVE-2025-15367" # No fix available yet
"CVE-2025-12781" # No fix available yet
"CVE-2025-11468" # No fix available yet
"CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
"CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
"GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
"GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-25639" # axios DoS via __proto__ in mergeConfig - transitive dev dep via @neondatabase/api-client, not imported in application code
"CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
"GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet
"CVE-2026-3644" # Python 3.13 vulnerability - no fix available in base image
"CVE-2026-4224" # Python 3.13 Expat parser stack overflow in ElementDeclHandler - no fix available in base image
)
# Build JSON array of allowlisted CVE IDs for jq
ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .)
echo "Checking for vulnerabilities with CVSS score >= 4.0..."
echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
echo ""
# Show all high-severity vulnerabilities for transparency
TOTAL_HIGH_SEVERITY=$(grype litellm:latest -o json | jq -r '
.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| .vulnerability.id' | wc -l)
if [ "$TOTAL_HIGH_SEVERITY" -gt 0 ]; then
echo "Total vulnerabilities found with CVSS >= 4.0: $TOTAL_HIGH_SEVERITY"
echo ""
echo "All high-severity vulnerabilities (including allowlisted):"
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Allowlisted"],
(.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, (if (.vulnerability.id as $id | $allow | index($id)) then "YES" else "NO" end)])
| @tsv' | column -t -s $'\t'
echo ""
fi
HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| .vulnerability.id' | wc -l)
if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
echo ""
echo "=========================================="
echo "ERROR: Security Scan Failed"
echo "=========================================="
echo "Found $HIGH_SEVERITY_COUNT non-allowlisted vulnerabilities with CVSS score >= 4.0 in litellm:latest"
echo ""
echo "These vulnerabilities are NOT in the allowlist and must be addressed."
echo "Current allowlisted CVEs: ${ALLOWED_CVES[*]}"
echo ""
echo "Detailed vulnerability report:"
echo ""
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
(.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
| @tsv' | column -t -s $'\t'
echo ""
echo "=========================================="
echo "Action Required:"
echo "=========================================="
echo "1. If a fix is available, update the package to the fixed version"
echo "2. If the vulnerability is not applicable or has no fix:"
echo " - Add the CVE/GHSA ID to ALLOWED_CVES array in ci_cd/security_scans.sh"
echo " - Add a comment explaining why it's safe to ignore"
echo ""
echo "Note: Some vulnerabilities may have multiple IDs (CVE-XXXX and GHSA-XXXX)."
echo "Add all relevant IDs to the allowlist if they refer to the same issue."
echo "=========================================="
echo ""
exit 1
else
echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"
fi
echo "Grype scans completed successfully"
}
# Main execution
main() {
echo "Installing security scanning tools..."
install_trivy
install_grype
# echo "Running secret detection scans..."
# run_secret_detection
echo "Running filesystem vulnerability scans..."
run_trivy_scans
echo "Running Docker image vulnerability scans..."
run_grype_scans
echo "All security scans completed successfully!"
}
# Execute main function
main "$@"

8
docker/Dockerfile.custom_ui
View file

@ -71,8 +71,16 @@ WORKDIR /app
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
# Run as non-root user
RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser \
&& chown -R appuser:appuser /app
USER appuser
# Expose the necessary port
EXPOSE 4000/tcp
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health')"]
# Override the CMD instruction with your desired command and arguments
CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]

1
docker/Dockerfile.dev
View file

@ -15,6 +15,7 @@ USER root
# Install build dependencies in one layer
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
g++ \
python3-dev \
libssl-dev \
pkg-config \

8
docker/Dockerfile.health_check
View file

@ -13,12 +13,12 @@ RUN pip install --no-cache-dir -r requirements.txt
RUN chmod +x /app/health_check_client.py
# Run as non-root user
RUN adduser --disabled-password --gecos "" --uid 1001 healthcheck
USER healthcheck
RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser
USER appuser
# Health check
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
CMD python /app/health_check_client.py --help || exit 1
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
CMD ["python", "/app/health_check_client.py", "--help"]
# Set entrypoint
ENTRYPOINT ["python", "/app/health_check_client.py"]

20
docker/Dockerfile.non_root
View file

@ -41,19 +41,24 @@ COPY . .
ENV LITELLM_NON_ROOT=true
# Build Admin UI using the upstream command order while keeping a single RUN layer
# NOTE: .npmrc (which has ignore-scripts=true and min-release-age=3d) is temporarily
# renamed during npm install/ci. This is safe because npm ci installs from
# NOTE: .npmrc files (which may set ignore-scripts=true and min-release-age=3d)
# are temporarily renamed during npm install/ci so they don't block lifecycle
# scripts needed by the build. This is safe because npm ci installs from
# package-lock.json with pinned versions + integrity hashes.
RUN mkdir -p /var/lib/litellm/ui && \
mv /app/.npmrc /app/.npmrc.bak && \
npm install -g npm@11.12.1 && npm cache clean --force && \
([ -f /app/.npmrc ] && mv /app/.npmrc /app/.npmrc.bak || true) && \
npm install -g npm@11.12.1 && \
npm install -g node-gyp@12.2.0 && \
ln -sf "$(npm root -g)/node-gyp" "$(npm root -g)/npm/node_modules/node-gyp" && \
npm cache clean --force && \
cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
mv .npmrc .npmrc.bak && \
([ -f .npmrc ] && mv .npmrc .npmrc.bak || true) && \
npm ci && \
mv .npmrc.bak .npmrc && mv /app/.npmrc.bak /app/.npmrc && \
([ -f .npmrc.bak ] && mv .npmrc.bak .npmrc || true) && \
([ -f /app/.npmrc.bak ] && mv /app/.npmrc.bak /app/.npmrc || true) && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
mkdir -p /var/lib/litellm/assets && \
@ -139,6 +144,9 @@ COPY --from=builder /app/requirements.txt /app/requirements.txt
COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
COPY --from=builder /app/schema.prisma /app/
# Keep enterprise bridge module in runtime so `enterprise.enterprise_hooks`
# can load and register managed enterprise hooks (e.g. managed_files).
COPY --from=builder /app/enterprise /app/enterprise
# Copy prisma_migration.py for Helm migrations job compatibility
COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
COPY --from=builder /wheels/ /wheels/

8
docker/README.md
View file

@ -13,19 +13,19 @@ To build and run the application, you will use the `docker-compose.yml` file loc
### 1. Set the Master Key
The application requires a `MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application.
The application requires a `LITELLM_MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application.
Create a `.env` file in the root of the project and add the following line:
```
MASTER_KEY=your-secret-key
LITELLM_MASTER_KEY=your-secret-key
```
Replace `your-secret-key` with a strong, randomly generated secret.
### 2. Build and Run the Containers
Once you have set the `MASTER_KEY`, you can build and run the containers using the following command:
Once you have set the `LITELLM_MASTER_KEY`, you can build and run the containers using the following command:
```bash
docker compose up -d --build
@ -89,4 +89,4 @@ This command should succeed (showing engine versions) even with `--network none`
## Troubleshooting
- **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.
- **`Master key is not initialized`**: This error means the `MASTER_key` environment variable is not set. Make sure you have created a `.env` file in the project root with the `MASTER_KEY` defined.
- **`Master key is not initialized`**: This error means the `LITELLM_MASTER_KEY` environment variable is not set. Make sure you have created a `.env` file in the project root with the `LITELLM_MASTER_KEY` defined.

7
docs/my-website/.trivyignore
View file

@ -1,7 +0,0 @@
# js-yaml CVE-2025-64718
# This vulnerability is not applicable because we've forced js-yaml to version 4.1.1
# via npm overrides in package.json. Trivy incorrectly reports this based on
# dependency requirements in the lockfile, but the actual installed version is 4.1.1.
# Verified with: npm list js-yaml
CVE-2025-64718

39
docs/my-website/blog/april_townhall_announcement/index.md Normal file
View file

@ -0,0 +1,39 @@
---
slug: april-townhall-announcement
title: "April Townhall: Security + Product Roadmap"
date: 2026-04-02T07:30:00
authors:
- krrish
description: "Join the LiteLLM April townhall on Friday, 10 April at 7:30 AM to learn about LiteLLM's security and product roadmap."
tags: [announcement, townhall]
hide_table_of_contents: true
---
import Image from '@theme/IdealImage';
We are hosting our April townhall on **Friday, 10 April at 7:30 AM PST**.
<Image
img={require('../../img/april_townhall_banner.png')}
style={{width: '900px', height: 'auto', display: 'block'}}
/>
{/* truncate */}
## Agenda
- Product updates and roadmap progress
- Reliability and security updates
- Open Q&A with the team
## How to contribute
Add your thoughts to this [ticket](https://github.com/BerriAI/litellm/issues/24825) to help us shape the agenda.
## Register
Register here: [LiteLLM April Townhall Form](https://forms.gle/hvyVXwbFjzJQE7dEA)
We will hold the townhall from **7:30 AM to 8:30 AM PST on Zoom**.
For security, attendance is restricted to corporate emails. If you register with a non-corporate email, we will share the townhall slides and accompanying blog post after the event.

35
docs/my-website/blog/ci_cd_v2_improvements/index.md
View file

@ -27,6 +27,41 @@ Building on the roadmap from our [security incident](https://docs.litellm.ai/blo
- Validation and release are separated into different repositories, making it harder for an attacker to reach release credentials.
- Trusted Publishing for PyPI releases - this means no long-lived credentials are used to publish releases.
- Immutable Docker release tags - this means no tampering of Docker release tags after they are published [Learn more](https://docs.docker.com/docker-hub/repos/manage/hub-images/immutable-tags/). Note: work for GHCR docker releases is planned as well.
- Docker image signing with [Cosign](https://github.com/sigstore/cosign) - all release images are signed so users can independently verify they came from us.
## Verify Docker image signatures
Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
**Verify using a release tag (convenience):**
Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
Expected output:
```
The following checks were performed on each of these signatures:
- The cosign claims were validated
- The signatures were verified against the specified public key
```
## What's next

66
docs/my-website/blog/security_hardening_april_2026/index.md Normal file
View file

@ -0,0 +1,66 @@
---
slug: security-hardening-april-2026
title: "Security Update: Vulnerability Disclosures and Ongoing Hardening"
date: 2026-04-03T12:00:00
authors:
- krrish
- ishaan-alt
description: "Disclosure of security vulnerabilities fixed in LiteLLM v1.83.0, and the launch of our bug bounty program."
tags: [security]
hide_table_of_contents: false
---
After the [supply chain incident](https://docs.litellm.ai/blog/security-update-march-2026) in March, we brought in [Veria Labs](https://verialabs.com/) to audit the LiteLLM proxy and fixed a number of vulnerability reports from independent researchers. All issues below are fixed in v1.83.0. If you are affected, particularly if you have JWT auth enabled, we recommend upgrading.
We've also launched a [bug bounty program](#bug-bounty-program) and Veria Labs is continuing to audit the proxy. More fixes will ship in upcoming versions.
The two high-severity issues ([CVE-2026-35029](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789) and [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)) **both require the attacker to already have a valid API key for the proxy**. These are not exploitable by unauthenticated users.
The critical-severity issue ([CVE-2026-35030](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)) is an authentication bypass, but only affects deployments with `enable_jwt_auth` explicitly enabled, which is off by default. **The default LiteLLM configuration is not affected, and no LiteLLM Cloud customers had this feature enabled.**
{/* truncate */}
## Vulnerabilities
### CVE-2026-35030: Authentication bypass via OIDC cache collision (Critical)
Found by Veria Labs.
When `enable_jwt_auth` is enabled, LiteLLM cached OIDC userinfo using `token[:20]` as the cache key. JWTs from the same signing algorithm share the same header prefix, so an attacker could forge a token that hits another user's cache entry and inherit their session. We fixed this by keying the cache on `sha256(token)` instead.
**Most deployments are not affected.** This requires `enable_jwt_auth: true`, which is off by default. If you can't upgrade, disable JWT auth as a workaround.
Full advisory: [GHSA-jjhc-v7c2-5hh6](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)
### CVE-2026-35029: Privilege escalation via `/config/update` (High)
Found by Lakera.
`/config/update` didn't check the caller's role. Any authenticated user could modify the proxy's runtime configuration, which could lead to arbitrary file read, admin account takeover, or remote code execution. We now require the `proxy_admin` role on this endpoint.
Full advisory: [GHSA-53mr-6c8q-9789](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789)
### Password hash exposure and pass-the-hash login (High)
Weak hashing originally reported by GitHub user [hamzayevmaqsud](https://github.com/hamzayevmaqsud) ([#15484](https://github.com/BerriAI/litellm/issues/15484)). The full chain was identified by Luca Vandenweghe and Maarten De Rammelaere of [iO Digital](https://www.iodigital.com/).
Passwords were stored as unsalted SHA-256 hashes, and in some cases plaintext. Several API endpoints returned the hash to any authenticated user, and `/v2/login` accepted the raw hash as a credential without re-hashing it, so a stolen hash was as good as the password itself. We've moved to scrypt with random salts and stripped hashes from all API responses.
Full advisory: [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)
## Bug bounty program
After the supply chain incident and these disclosures it was clear we needed more external eyes on the project. We've set up a bug bounty program so researchers have a way to report issues.
Bounties are currently paid for P0 (supply chain) and P1 (unauthenticated proxy access) vulnerabilities:
| Severity | Bounty | Example |
|----------|--------|---------|
| Critical | $1,500 – $3,000 | Supply chain compromise |
| High | $500 – $1,500 | Unauthenticated access to protected data |
We plan on expanding the program further in the coming months. More info about the bug bounty program is available [here](https://github.com/BerriAI/litellm/security).
## What's next
Veria Labs is continuing to work with us on a broader audit of the proxy. Security advisories sent through Github will be responded to within five business days. We'll publish advisories as issues are confirmed and fixed.

35
docs/my-website/blog/security_townhall_updates/index.md
View file

@ -143,8 +143,41 @@ This will ensure, your releases are safe, even when:
- Tampered registry artifacts are published
- Tag mutations are made after the release is published
We believe that [Cosign](https://github.com/sigstore/cosign) is a good fit for this, and have already begun working on it [PR](https://github.com/BerriAI/litellm/pull/24683).
We believe that [Cosign](https://github.com/sigstore/cosign) is a good fit for this, and have shipped it in [PR #24683](https://github.com/BerriAI/litellm/pull/24683).
#### How to verify a Docker image with Cosign
Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key that was introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
**Verify using a release tag (convenience):**
Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
Expected output:
```
The following checks were performed on each of these signatures:
- The cosign claims were validated
- The signatures were verified against the specified public key
```
### Avoid Compromised Packages

34
docs/my-website/blog/security_update_march_2026/index.md
View file

@ -708,6 +708,40 @@ The LiteLLM AI Gateway team has already taken the following steps:
- Engaged Google's Mandiant security team to assist with forensic analysis of the build and publishing chain
## Verify Docker image signatures
Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
**Verify using a release tag (convenience):**
Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
Expected output:
```
The following checks were performed on each of these signatures:
- The cosign claims were validated
- The signatures were verified against the specified public key
```
## Verified safe versions
We have audited every LiteLLM release published between v1.78.0 and v1.82.6 across both PyPI and Docker. Each artifact was verified by:

3
docs/my-website/docs/mcp.md
View file

@ -278,7 +278,8 @@ mcp_servers:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_role_name: os.environ/AWS_ROLE_ARN # optional — IAM role to assume
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # optional — falls back to IAM role
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
aws_service_name: bedrock-agentcore

53
docs/my-website/docs/mcp_aws_sigv4.md
View file

@ -36,6 +36,8 @@ LiteLLM's `aws_sigv4` auth type handles this automatically: every outgoing MCP r
| **AWS Access Key ID** | No | Falls back to boto3 credential chain if blank |
| **AWS Secret Access Key** | No | Required if Access Key ID is provided |
| **AWS Session Token** | No | Only needed for temporary STS credentials |
| **AWS Role ARN** | No | IAM role ARN for STS AssumeRole (e.g., `arn:aws:iam::123456789012:role/MyRole`). If set, LiteLLM assumes this role before signing |
| **AWS Session Name** | No | Session name for the AssumeRole call — appears in CloudTrail. Auto-generated if omitted |
Once created, LiteLLM will sign every outgoing MCP request with SigV4. The server's tools appear automatically in the MCP Tools list.
@ -66,8 +68,8 @@ mcp_servers:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_role_name: os.environ/AWS_ROLE_ARN # IAM role to assume (recommended)
aws_session_name: "litellm-prod" # optional — for CloudTrail auditing
aws_region_name: "us-east-1"
aws_service_name: "bedrock-agentcore"
```
@ -128,6 +130,8 @@ curl http://localhost:4000/mcp-rest/tools/call \
| `aws_region_name` | Yes | AWS region (e.g., `us-east-1`) |
| `aws_service_name` | No | AWS service name for signing. Defaults to `bedrock-agentcore` |
| `aws_session_token` | No | AWS session token for temporary credentials. Supports `os.environ/VAR_NAME` |
| `aws_role_name` | No | IAM role ARN for STS AssumeRole. Supports `os.environ/VAR_NAME`. When set, LiteLLM calls `sts:AssumeRole` to get temporary credentials before signing |
| `aws_session_name` | No | Session name for the AssumeRole call (appears in CloudTrail). Auto-generated if omitted. Supports `os.environ/VAR_NAME` |
## How It Works
@ -157,6 +161,42 @@ mcp_servers:
aws_service_name: "bedrock-agentcore"
```
## Using IAM Role Assumption (AssumeRole)
For production environments where your LiteLLM instance authenticates via an IAM role (e.g., EKS pod role, EC2 instance profile), you can configure `aws_role_name` to have LiteLLM call `sts:AssumeRole` before signing MCP requests:
```yaml title="config.yaml with AssumeRole" showLineNumbers
mcp_servers:
my_agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_role_name: "arn:aws:iam::123456789012:role/BedrockAgentCoreRole"
aws_session_name: "litellm-prod" # optional
aws_region_name: "us-east-1"
aws_service_name: "bedrock-agentcore"
```
LiteLLM uses the ambient credentials (pod role, instance profile, or env vars) to call `sts:AssumeRole`, then signs MCP requests with the assumed role's temporary credentials.
You can also combine `aws_role_name` with explicit access keys — the keys are then used as the source identity for the AssumeRole call:
```yaml title="config.yaml with AssumeRole + explicit source keys" showLineNumbers
mcp_servers:
my_agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_role_name: os.environ/AWS_ROLE_ARN
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: "us-east-1"
```
:::tip
For most Kubernetes deployments, you only need `aws_role_name` and `aws_region_name` — the pod's IAM role provides the source credentials automatically.
:::
## Troubleshooting
### 403 Forbidden from AWS
@ -166,6 +206,15 @@ mcp_servers:
- Ensure `aws_service_name` is set to `bedrock-agentcore`
- If using STS credentials, confirm `aws_session_token` is set and not expired
### AssumeRole AccessDenied
If you get `AccessDenied` when using `aws_role_name`:
- Verify the role ARN is correct
- Check that the trust policy on the target role allows your source identity to assume it
- If running on EKS, ensure the pod's service account is annotated with the correct IAM role
- Check CloudTrail for the failed `sts:AssumeRole` call to see the exact error
### Health check errors on startup
SigV4-authenticated MCP servers skip the standard health check on proxy startup. This is expected — the proxy will still sign requests correctly when tools are invoked.

231
docs/my-website/docs/mcp_toolsets.md Normal file
View file

@ -0,0 +1,231 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# MCP Toolsets
A **Toolset** is a named collection of specific tools drawn from one or more MCP servers. Instead of giving an agent access to every tool on every server, you pick exactly which tools it needs — from whichever servers they live on — and bundle them under a single name.
## How it works
```
┌─────────────────────────────────┐
│ MCP Toolset │
│ "devtooling-prod" │
└────────────┬────────────────────┘
│
┌──────────────────┴──────────────────┐
│ │
┌────────▼────────┐ ┌────────▼────────┐
│ CircleCI MCP │ │ DeepWiki MCP │
│ (10+ tools) │ │ (3 tools) │
└────────┬────────┘ └────────┬────────┘
│ │
┌─────────┴──────────┐ ┌──────────┴──────────┐
│ ✓ get_build_logs │ │ ✓ read_wiki_structure│
│ ✓ find_flaky_tests │ │ ✓ read_wiki_contents │
│ ✓ get_pipeline_ │ │ ✗ ask_question │
│ status │ └─────────────────────┘
│ ✓ run_pipeline │
│ ✗ list_followed_ │
│ projects │
└────────────────────┘
Agent sees exactly 6 tools, nothing more.
```
Instead of 13+ tools across two servers, the agent gets 6 — the ones it actually needs.
**Why this matters:**
- Smaller tool lists → fewer tokens, faster responses, less hallucination
- Combine tools from GitHub + Linear + CircleCI into one named grant
- Assign to keys and teams the same way you assign MCP servers today
---
## Create a toolset
### 1. Go to the MCP page
Navigate to **MCP** in the left sidebar.
![Navigate to MCP](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/1a96c713-6a37-4f96-92f1-07bd58c1973c/ascreenshot_23515f386ccc4597b0633987667fe01f_text_export.jpeg)
### 2. Open the Toolsets tab
Click the **Toolsets** tab on the MCP page.
![Click Toolsets tab](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/65b6986b-595a-4b28-8fdc-a7b36bc76e59/ascreenshot_ca70c18fe7ec415486f96a6b405bf550_text_export.jpeg)
### 3. Click "New Toolset"
![New Toolset button](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/798c55c4-5d6b-4815-a642-70ac9f34f102/ascreenshot_3f144f54a1a944e28454239c837b4e6d_text_export.jpeg)
### 4. Enter a name
Type a name for the toolset. Pick something descriptive — this is what agents will reference.
![Enter toolset name](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/62b412e0-d38f-44c3-99e4-3693f1512f6a/ascreenshot_b678c7c988a04f8b887b0f54c4dd95a7_text_export.jpeg)
![Toolset name field](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/ba5ebc95-cab7-470b-a7c9-21f12b9b01a3/ascreenshot_a602e982a2a44890a83dca64d61c38eb_text_export.jpeg)
### 5. Add the first tool
Select an MCP server from the dropdown, then choose the tool you want to include from that server.
![Select MCP server](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/2aa5bcba-6414-42e3-9813-efb0a9078e32/ascreenshot_58fbff35ba654210a1b4dc5452aa6bd9_text_export.jpeg)
![Choose server from dropdown](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/4fd9cffb-d3ba-461a-8679-89f278bf67ad/ascreenshot_b61e9e85a51b494a8d09fe61198d63e1_text_export.jpeg)
![Select tool from server](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/60718e72-2062-494b-9a23-456992c88cbd/ascreenshot_7a1f8eeab30a4a05ba39c450e5458b78_text_export.jpeg)
### 6. Add tools from a second server
Click **Add Tool**, pick a different MCP server, and select another tool. Repeat for as many tools as you need — they can come from any number of servers.
![Add tool from second server](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/f34e0600-cc74-4b18-8794-88d45f326144/ascreenshot_98834b14ab9343e39fb503e458d72b7c_text_export.jpeg)
![Select second server](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/75150368-2202-4da1-99f1-6f0620e9b133/ascreenshot_f94d0bc08ea147348a9cf021cce7d854_text_export.jpeg)
![Select tool from second server](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/ed2cdf6e-025d-4d50-8b12-ed68745d5c51/ascreenshot_0c1c7f76524b46c5a056fda5e6956e2b_text_export.jpeg)
### 7. Create the toolset
Click **Create Toolset** to save.
![Create Toolset](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/021ca7b3-2d9a-49a0-8758-dae3dc3bcb4d/ascreenshot_14c6434e71114a6091e359a996f20e12_text_export.jpeg)
---
## Use a toolset in the Playground
Once created, your toolset appears alongside MCP servers in the **MCP Servers** dropdown in the Playground — it's selectable the same way.
### 1. Go to the Playground
![Navigate to Playground](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/f9d4aa4c-d98e-4767-b98e-aad2890e97ca/ascreenshot_d84239c441bb4e828f229d0c9e079e3f_text_export.jpeg)
![Click Playground](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/d8a07563-97fe-453a-b974-88da46c87294/ascreenshot_ea494300a536400abb2ea6bf3bdfd5ab_text_export.jpeg)
### 2. Select your toolset from MCP Servers
In the left panel under **MCP Servers**, open the dropdown and pick your toolset. The model will only see the tools you included in it.
![Select MCP servers dropdown](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/ee8cb38c-c4ff-4b4b-844c-22f2e40832ae/ascreenshot_e300fb39cea0434fb5e3986e912a2b8d_text_export.jpeg)
![Open MCP server picker](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/8672070c-5d07-4f63-878c-6fc7dcbc9b65/ascreenshot_326ddd0868224c99a6fa5dab2d144f1f_text_export.jpeg)
![Select toolset](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/955826ad-2bbb-403e-ab26-c1ac03ec2675/ascreenshot_13f837ad53574535986ca7ca5998d34a_text_export.jpeg)
![Toolset selected and active](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/9a59c3b9-1563-4731-838f-1c35d636ddc9/ascreenshot_c05d8fa5f37a4b3093fc46e26f293b4d_text_export.jpeg)
The model now has access to exactly the tools in your toolset and nothing else.
---
## Use a toolset via API
Pass the toolset's route as the `server_url` in your tools list. LiteLLM resolves it server-side — no public URL needed.
<Tabs>
<TabItem value="responses" label="Responses API">
```python
import openai
client = openai.OpenAI(
api_key="your-litellm-key",
base_url="http://your-proxy/v1",
)
response = client.responses.create(
model="gpt-4o",
input="What CI/CD tools do you have?",
tools=[
{
"type": "mcp",
"server_label": "devtooling-prod",
"server_url": "litellm_proxy/mcp/devtooling-prod",
"require_approval": "never",
}
],
)
print(response.output_text)
```
</TabItem>
<TabItem value="chat" label="Chat Completions API">
```python
import openai
client = openai.OpenAI(
api_key="your-litellm-key",
base_url="http://your-proxy/v1",
)
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": "What CI/CD tools do you have?"}],
tools=[
{
"type": "mcp",
"server_label": "devtooling-prod",
"server_url": "litellm_proxy/mcp/devtooling-prod",
"require_approval": "never",
}
],
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="rest" label="REST">
```bash
curl http://your-proxy/v1/responses \
-H "Authorization: Bearer your-litellm-key" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o",
"input": "What CI/CD tools do you have?",
"tools": [
{
"type": "mcp",
"server_label": "devtooling-prod",
"server_url": "litellm_proxy/mcp/devtooling-prod",
"require_approval": "never"
}
]
}'
```
</TabItem>
</Tabs>
---
## Manage toolsets via API
```bash
# List all toolsets
curl http://your-proxy/v1/mcp/toolset \
-H "Authorization: Bearer your-litellm-key"
# Create a toolset
curl -X POST http://your-proxy/v1/mcp/toolset \
-H "Authorization: Bearer your-litellm-key" \
-H "Content-Type: application/json" \
-d '{
"toolset_name": "devtooling-prod",
"description": "CircleCI + DeepWiki tools for the dev team",
"tools": [
{"server_id": "<circleci-server-id>", "tool_name": "get_build_failure_logs"},
{"server_id": "<circleci-server-id>", "tool_name": "run_pipeline"},
{"server_id": "<deepwiki-server-id>", "tool_name": "read_wiki_structure"}
]
}'
# Delete a toolset
curl -X DELETE http://your-proxy/v1/mcp/toolset/<toolset_id> \
-H "Authorization: Bearer your-litellm-key"
```

131
docs/my-website/docs/observability/ramp_integration.md Normal file
View file

@ -0,0 +1,131 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Ramp
Send AI usage and cost data to Ramp for automated spend tracking.
[Ramp](https://ramp.com/) is a finance automation platform that helps businesses manage expenses, corporate cards, and vendor payments. With the Ramp callback integration, your LiteLLM AI usage — including token counts, model costs, and request metadata — is automatically sent to Ramp for real-time spend visibility.
:::info
We want to learn how we can make the callbacks better! Meet the LiteLLM [founders](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) or
join our [discord](https://discord.gg/wuPM9dRgDw)
:::
## Pre-Requisites
1. Log in to [Ramp](https://app.ramp.com/) and search for **"LiteLLM"** using the search bar. Click the **LiteLLM** integration result.
> **Note:** Only business owners and admins can access and configure integrations.
2. On the LiteLLM integration page, click the **Connect** button in the top right.
3. In the Connect LiteLLM drawer, click **Generate API Key** to create an API key.
> **Important:** Copy the API key immediately — it won't be shown again. If you lose it, you can revoke the existing key and generate a new one from the integration settings.
```shell
pip install litellm
```
## Quick Start
Set your `RAMP_API_KEY` and add `"ramp"` to your callbacks to start logging LLM usage to Ramp.
<Tabs>
<TabItem value="python" label="SDK">
```python
litellm.callbacks = ["ramp"]
```
```python
import litellm
import os
# Ramp API Key
os.environ["RAMP_API_KEY"] = "your-ramp-api-key"
# LLM API Keys
os.environ['OPENAI_API_KEY'] = ""
# Set ramp as a callback
litellm.callbacks = ["ramp"]
# OpenAI call
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[
{"role": "user", "content": "Hi - I'm testing Ramp integration"}
]
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
callbacks: ["ramp"]
environment_variables:
RAMP_API_KEY: os.environ/RAMP_API_KEY
```
2. Start LiteLLM Proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "Hey, how are you?"
}
]
}'
```
</TabItem>
</Tabs>
## What Data is Logged?
LiteLLM sends the [Standard Logging Payload](https://docs.litellm.ai/docs/proxy/logging_spec) to Ramp on successful LLM API calls, which includes:
- **Request details**: Model, messages, parameters
- **Response details**: Completion text, token usage, latency
- **Metadata**: User ID, custom metadata, timestamps
- **Cost tracking**: Response cost based on token usage
## Authentication
Set the `RAMP_API_KEY` environment variable with your Ramp API key.
| Environment Variable | Description |
|---|---|
| `RAMP_API_KEY` | Your Ramp API key (required) |
## Support & Talk to Founders
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

4
docs/my-website/docs/providers/azure/azure_responses.md
View file

@ -246,7 +246,7 @@ You can also call the Azure Responses API via the `/chat/completions` endpoint.
from litellm import completion
import os
os.environ["AZURE_API_BASE"] = "https://my-endpoint-sweden-berri992.openai.azure.com/"
os.environ["AZURE_API_BASE"] = "https://my-azure-endpoint.openai.azure.com/"
os.environ["AZURE_API_VERSION"] = "2023-03-15-preview"
os.environ["AZURE_API_KEY"] = "my-api-key"
@ -268,7 +268,7 @@ model_list:
litellm_params:
model: azure/responses/my-custom-o1-pro
api_key: os.environ/AZURE_API_KEY
api_base: https://my-endpoint-sweden-berri992.openai.azure.com/
api_base: https://my-azure-endpoint.openai.azure.com/
api_version: 2023-03-15-preview
```

24
docs/my-website/docs/providers/bedrock_image_gen.md
View file

@ -111,6 +111,29 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
</TabItem>
</Tabs>
## Amazon Nova Canvas - Image Edit
Use OpenAI-compatible `image_edit()` with Bedrock Nova Canvas (`amazon.nova-canvas-v1:0`). Requests use the same `InvokeModel` API as generation; LiteLLM maps inputs to [Nova Canvas task types](https://docs.aws.amazon.com/nova/latest/userguide/image-gen-access.html):
| Scenario | `taskType` sent to Bedrock |
|----------|----------------------------|
| Image + prompt (no mask) | `IMAGE_VARIATION` |
| Image + prompt + mask | `INPAINTING` (`inPaintingParams.image`, `maskImage` or `maskPrompt`) |
| `taskType: OUTPAINTING` + `mask` or `maskPrompt` | `OUTPAINTING` (Bedrock requires one; LiteLLM raises a clear error if both are missing) |
| `taskType: BACKGROUND_REMOVAL` | `BACKGROUND_REMOVAL` |
```python
from litellm import image_edit
response = image_edit(
image=open("photo.png", "rb"),
prompt="Add soft sunset lighting",
model="bedrock/amazon.nova-canvas-v1:0",
)
```
For **`BACKGROUND_REMOVAL`**, the AWS request must not include `imageGenerationConfig`; LiteLLM omits it for that task even if you pass `size`, `n`, `seed`, etc. Additional Nova Canvas inference IDs for image edit should set **`supports_nova_canvas_image_edit`: true** in `model_prices_and_context_window.json` (see `amazon.nova-canvas-v1:0`).
## Using Inference Profiles with Image Generation
For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN:
@ -147,4 +170,3 @@ model_list:
## Authentication
All standard Bedrock authentication methods are supported for image generation. See [Bedrock Authentication](./bedrock#boto3---authentication) for details.

16
docs/my-website/docs/providers/gemini.md
View file

@ -65,14 +65,13 @@ response = completion(
- modalities
- reasoning_content
- audio (for TTS models only)
- service_tier
**Anthropic Params**
- thinking (used to set max budget tokens across anthropic/gemini models)
[**See Updated List**](https://github.com/BerriAI/litellm/blob/main/litellm/llms/gemini/chat/transformation.py#L70)
## Usage - Thinking / `reasoning_content`
LiteLLM translates OpenAI's `reasoning_effort` to Gemini's `thinking` parameter. [Code](https://github.com/BerriAI/litellm/blob/620664921902d7a9bfb29897a7b27c1a7ef4ddfb/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py#L362)
@ -298,6 +297,19 @@ curl http://0.0.0.0:4000/v1/chat/completions \
## Usage - `service_tier`
LiteLLM propagates OpenAI's `service_tier` parameter to Gemini, and also extracts it from the response headers (`x-gemini-service-tier`) into `model_response.service_tier`.
| OpenAI `service_tier` | Gemini `service_tier` | Notes |
| --------------------- | --------------------- | ----- |
| `"auto"` | `"priority"` | LiteLLM maps OpenAI's `"auto"` to Gemini's `"priority"` tier, as `priority` will fall back on Gemini. |
| `"flex"` | `"flex"` | Direct mapping. |
| `"priority"` | `"priority"` | Direct mapping. |
| `"default"` | `"standard"` | LiteLLM maps `"default"` to `"standard"`. |
| Any other value | Passed as-is (lowercased) | Values are case-insensitive and normalized to lowercase. |
On the response, LiteLLM maps `"standard"` back to `"default"` for the Gemini API.
## Text-to-Speech (TTS) Audio Output

109
docs/my-website/docs/providers/oci.md
View file

@ -8,24 +8,54 @@ Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generativ
## Supported Models
### Meta Llama Models
### Chat / Text Generation
#### Meta Llama Models
- `meta.llama-4-maverick-17b-128e-instruct-fp8`
- `meta.llama-4-scout-17b-16e-instruct`
- `meta.llama-3.3-70b-instruct`
- `meta.llama-3.3-70b-instruct-fp8-dynamic`
- `meta.llama-3.2-90b-vision-instruct`
- `meta.llama-3.2-11b-vision-instruct`
- `meta.llama-3.1-405b-instruct`
- `meta.llama-3.1-70b-instruct`
### xAI Grok Models
#### xAI Grok Models
- `xai.grok-4.20`
- `xai.grok-4.20-multi-agent`
- `xai.grok-4`
- `xai.grok-4-fast`
- `xai.grok-4.1-fast`
- `xai.grok-3`
- `xai.grok-3-fast`
- `xai.grok-3-mini`
- `xai.grok-3-mini-fast`
- `xai.grok-code-fast-1`
### Cohere Models
#### Cohere Models
- `cohere.command-latest`
- `cohere.command-a-03-2025`
- `cohere.command-a-reasoning-08-2025`
- `cohere.command-a-vision-07-2025`
- `cohere.command-a-translate-08-2025`
- `cohere.command-plus-latest`
- `cohere.command-r-08-2024`
- `cohere.command-r-plus-08-2024`
#### Google Gemini Models (via OCI)
- `google.gemini-2.5-pro`
- `google.gemini-2.5-flash`
- `google.gemini-2.5-flash-lite`
### Embedding Models
- `cohere.embed-english-v3.0` (1024 dimensions)
- `cohere.embed-english-light-v3.0` (384 dimensions)
- `cohere.embed-multilingual-v3.0` (1024 dimensions)
- `cohere.embed-multilingual-light-v3.0` (384 dimensions)
- `cohere.embed-english-image-v3.0` (1024 dimensions, multimodal)
- `cohere.embed-english-light-image-v3.0` (384 dimensions, multimodal)
- `cohere.embed-multilingual-light-image-v3.0` (384 dimensions, multimodal)
- `cohere.embed-v4.0` (1536 dimensions, multimodal)
## Authentication
@ -394,4 +424,75 @@ response = completion(
| `oci_tenancy` | string | - | (Manual auth) The OCID of your OCI tenancy |
| `oci_key` | string | - | (Manual auth) The private key content as a string |
| `oci_key_file` | string | - | (Manual auth) Path to the private key file |
| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication |
| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication |
## Embeddings
LiteLLM supports OCI Generative AI embedding models. These models use the same authentication methods described above.
<Tabs>
<TabItem value="embed-manual" label="Manual Credentials" default>
```python
from litellm import embedding
response = embedding(
model="oci/cohere.embed-english-v3.0",
input=["Hello world", "Goodbye world"],
oci_region="us-ashburn-1",
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
oci_key=<string_with_content_of_oci_key>,
oci_compartment_id=<oci_compartment_id>,
)
print(response)
```
</TabItem>
<TabItem value="embed-sdk" label="OCI SDK Signer">
```python
from litellm import embedding
from oci.signer import Signer
signer = Signer(
tenancy="ocid1.tenancy.oc1..",
user="ocid1.user.oc1..",
fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx",
private_key_file_location="~/.oci/key.pem",
)
response = embedding(
model="oci/cohere.embed-english-v3.0",
input=["Hello world", "Goodbye world"],
oci_signer=signer,
oci_region="us-ashburn-1",
oci_compartment_id="<oci_compartment_id>",
)
print(response)
```
</TabItem>
</Tabs>
### Embedding Optional Parameters
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `input_type` | string | - | The type of input: `search_document`, `search_query`, `classification`, `clustering` |
| `truncate` | string | `END` | Truncation strategy when input exceeds max tokens: `END` or `START` |
### Using Dedicated Embedding Endpoints
```python
response = embedding(
model="oci/cohere.embed-english-v3.0",
input=["Hello world"],
oci_serving_mode="DEDICATED",
oci_endpoint_id="ocid1.generativeaiendpoint.oc1...",
oci_region="us-ashburn-1",
oci_compartment_id="<oci_compartment_id>",
# ... auth params
)
```

267
docs/my-website/docs/providers/sap.md
View file

@ -55,24 +55,33 @@ pip install litellm
```
### Step 2: Set Your Credentials
Choose **one** of these authentication methods:
> **Breaking change**: credential resolution is "first-source-wins"
>
> Credential resolution no longer merges individual fields across sources.
>
> Resolution order is:
`kwargs` → `service key` → `env (AICORE_*)` → `config` → `VCAP service`
>
> **Important behavior:** once LiteLLM finds *any* credential value in a source, it takes **all** credentials from that source exclusively (except `resource_group`, which may still be resolved separately).
Choose **one** of these authentication methods:
<Tabs>
<TabItem value="service-key" label="Service Key JSON (Recommended)">
<Tabs>
<TabItem value="service-key" label="Service Key JSON (Recommended)">
The simplest approach - paste your entire service key as a single environment variable.
The simplest approach - paste your entire service key as a single environment variable. The service key must be wrapped in a `credentials` object:
> **Note:** the service key no more needs to be wrapped in a "credentials" key.
```bash
export AICORE_SERVICE_KEY='{
"credentials": {
"clientid": "your-client-id",
"clientsecret": "your-client-secret",
"url": "https://<your-instance>.authentication.sap.hana.ondemand.com",
"serviceurls": {
"AI_API_URL": "https://api.ai.<your-region>.aws.ml.hana.ondemand.com"
}
}
}'
export AICORE_RESOURCE_GROUP="default"
```
@ -220,6 +229,17 @@ model="sap/gemini-2.5-pro"
# Incorrect - missing prefix
model="gpt-4o" # ❌ Won't work
```
3. **Environment variables** - Set the following list of credentials in .env file
<pre>
AICORE_AUTH_URL = "https://* * * .authentication.sap.hana.ondemand.com/oauth/token",
AICORE_CLIENT_ID = " *** ",
AICORE_CLIENT_SECRET = " *** ",
AICORE_RESOURCE_GROUP = " *** ",
AICORE_BASE_URL = "https://api.ai.***.cfapps.sap.hana.ondemand.com/v2"
</pre>
Other credential configuration options are also available. For more information, see the [SAP AI Core Documentation](https://help.sap.com/doc/generative-ai-hub-sdk/CLOUD/en-US/_reference/README_sphynx.html#configuration).
## Usage - LiteLLM Python SDK
### Proxy Usage
@ -506,6 +526,241 @@ response = embedding(
print(response.data[0]["embedding"]) # Vector representation
```
### Additional Modules
The SAP Gen AI Hub includes additional modules for advanced use cases:
- [Grounding](https://help.sap.com/docs/sap-ai-core/generative-ai/grounding-035c455a5a424697b60f4a24b6d791fe?locale=en-US)
- [Translation](https://help.sap.com/docs/sap-ai-core/generative-ai/translation?locale=en-US)
- [Data Masking](https://help.sap.com/docs/sap-ai-core/generative-ai/data-masking-d9a54d9ca54b40beacbd24e1663ec3b4?locale=en-US)
- [Content Filtering](https://help.sap.com/docs/sap-ai-core/generative-ai/content-filtering?locale=en-US)
#### Grounding
Grounding is a service designed to handle data-related tasks, such as grounding and retrieval, using vector databases. It provides specialized data retrieval through these databases, grounding the retrieval process with your own external and context-relevant data. Grounding combines generative AI capabilities with the ability to use real-time, precise data to improve decision-making and business operations for specific AI-driven business solutions.
##### Prerequisites
To use the Grounding module in the orchestration pipeline, you need to prepare the knowledge base in advance.
Generative AI hub offers multiple options for users to provide data (prepare a knowledge base):
- For Option 1: Upload the documents to a supported data repository and run the data pipeline to vectorize the documents.
- For Option 2: Provide the chunks of document via Vector API directly.
To use grounding, choose from one of the following options.
Usage example:
```python showLineNumbers title="Grounding Example"
from litellm import completion
grounding_config = {
'type': 'document_grounding_service',
'config': {
'filters': [
{'id': 's3-docs',
'data_repository_type': 'vector',
'search_config': {'max_chunk_count': 2},
'data_repositories': ['012345-6789-0123-4567-890123456789']
}
],
'placeholders': {'input': ['user_query'], 'output': 'grounding_response'},
'metadata_params': ['source', 'webUrl', 'title', 'mimeType', 'fileSuffix']
}
}
response = completion(model="sap/gpt-4o",
messages=[
{"content":"""Facility Solutions Company provides services to luxury residential complexes,
apartments, individual homes, and commercial properties such as office buildings, retail
spaces, industrial facilities, and educational institutions. Customers are encouraged to
reach out with maintenance requests, service deficiencies, follow-ups, or any issues they
need by email.""", "role": "system"},
{"content":"""You are a helpful assistant for any queries for answering questions.
Answer the request by providing relevant answers that fit to the request.
Request: {{ ?user_query }}
Context:{{ ?grounding_response }}""", "role": "user"}
],
placeholder_values={"user_query": "Is there a complaint?"},
grounding=grounding_config
)
print(response.choices[0].message.content)
```
For more information about all available grounding configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/using-grounding-module-e1c4dd100dfb42ab890e1d95f3516187?locale=en-US).
#### Translation
The translation module allows you to translate LLM text prompts into a chosen target language.
```python showLineNumbers title="Translation Example"
from litellm import completion
translation_config = {
'input':
{'type': 'sap_document_translation',
'config':
{'source_language': 'en-US',
'target_language': 'de-DE'}
},
'output':
{'type': 'sap_document_translation',
'config':
{'source_language': 'de-DE',
'target_language': 'fr-FR'}
}
}
response = completion(model="sap/gpt-4o",
messages=[{"role": "user", "content": "Hello world!"}],
translation=translation_config)
print(response.choices[0].message.content)
```
For more information about all available translation configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/translation?locale=en-US)
#### Data Masking
The data masking module serves to anonymize or pseudonymize personally identifiable information from the input for selected entities.
```python showLineNumbers title="Data Masking Example"
from litellm import completion, embedding
masking_config = {
'providers':
[
{
'type': 'sap_data_privacy_integration',
'method': 'anonymization',
'entities': [
{'type': 'profile-address'},
{'type': 'profile-email'},
{'type': 'profile-phone'},
{'type': 'profile-person'},
{'type': 'profile-location'}
]
}
]
}
mock_cv = "some text with personal information"
response = completion(model="sap/gpt-4o",
messages=[{"role": "user", "content": "Give a one sentence summary of the CV. CV: {{?cv}}?"}],
placeholder_values={"cv": mock_cv},
masking=masking_config)
print(response.choices[0].message.content)
# Data masking module also available for embedding
response = embedding(model="sap/text-embedding-3-small",
input=mock_cv,
masking=masking_config)
print(response.data[0])
```
For more information about all available data masking configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/enhancing-model-consumption-with-data-masking-66ad6f469afc4c2cbaa91a27a33f7b21?locale=en-US)
#### Content Filtering
The content filtering module allows you to filter input and output based on content safety criteria.
The module supports two services:
* Azure Content Safety
* Llama Guard 3
```python showLineNumbers title="Content Filtering Example"
from litellm import completion
filtering_config_azure = {
'input':
{
'filters':
[
{'type': 'azure_content_safety',
'config':
{'hate': 0,
'sexual': 0,
'violence': 0,
'self_harm': 0
}
}
]
},
'output':
{
'filters':
[
{'type': 'azure_content_safety',
'config': {'hate': 0,
'sexual': 0,
'violence': 0,
'self_harm': 0
}
}
]
}
}
response = completion(model="sap/gpt-4o",
messages=[{"role": "user", "content": "Hello world!"}],
filtering=filtering_config_azure)
print(response.choices[0].message.content)
# The model responds normally because the content does not violate any safety rules.
try:
response = completion(model="sap/gpt-4o",
messages=[{"role": "user", "content": "I hate you"}],
filtering=filtering_config_azure)
except Exception as e:
print(e)
# The service raises an error:
# "Input Filter: Content filtered due to safety violations. Please modify the prompt and try again."
```
For more information about all available content filtering configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/content-filtering?locale=en-US)
#### List of modules configuration for fallback
SAP GEN AI Hub supports a fallback mechanism for handling errors. This mechanism allows you to specify a list of fallback modules to use in case of errors. The fallback modules should contain all parameters that are required for configuring the request.
Required parameters:
- `model`
- `messages`
Optional parameters:
- `filtering`
- `grounding`
- `translation`
- `masking`
- `tools`
- and any of model's specific parameters.
```python showLineNumbers title="Fallback Example"
from litellm import completion
translation_config = {
'input':
{'type': 'sap_document_translation',
'config':
{'source_language': 'en-US',
'target_language': 'de-DE'}
},
'output':
{'type': 'sap_document_translation',
'config':
{'source_language': 'de-DE',
'target_language': 'fr-FR'}
}
}
response = completion(model="sap/gpt-4o",
messages=[{"role": "user", "content": "Hello world!"}],
translation=translation_config,
fallback_sap_modules=[{
"model":"sap/gemini-2.5-flash",
"messages":[{"role": "user", "content": "Hello world!"}],
"translation":translation_config
}])
# In case of error with the first configuration (model gpt-4o), the fallback module is used.
print(response.choices[0].message.content)
```
## Reference
### Supported Parameters

10
docs/my-website/docs/proxy/config_settings.md
View file

@ -201,6 +201,7 @@ router_settings:
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
| enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. |
| disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
| default_team_params | object | Default parameters applied to every new team created via `/team/new` (including SSO auto-created teams). Only fills in fields not explicitly set in the request. Sub-fields: `max_budget` (float), `budget_duration` (string, e.g. `"30d"`), `tpm_limit` (integer), `rpm_limit` (integer), `team_member_permissions` (array of strings, e.g. `["/team/daily/activity", "/key/generate"]`), `models` (array of strings — only applied to SSO auto-created teams). |
### general_settings - Reference
@ -237,7 +238,7 @@ router_settings:
| public_routes | List[str] | (Enterprise Feature) Control list of public routes |
| alert_types | List[str] | Control list of alert types to send to slack (Doc on alert types)[./alerting.md] |
| enforced_params | List[str] | (Enterprise Feature) List of params that must be included in all requests to the proxy |
| enable_oauth2_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication |
| enable_oauth2_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication on LLM + info routes |
| use_x_forwarded_for | str | If true, uses the X-Forwarded-For header to get the client IP address |
| service_account_settings | List[Dict[str, Any]] | Set `service_account_settings` if you want to create settings that only apply to service account keys (Doc on service accounts)[./service_accounts.md] |
| image_generation_model | str | The default model to use for image generation - ignores model set in request |
@ -288,6 +289,7 @@ router_settings:
| database_connection_pool_timeout | integer | Database connection pool timeout in seconds |
| disable_error_logs | boolean | If true, suppresses error tracking and storage in the database |
| enable_health_check_routing | boolean | If true, enables health check-driven request routing to avoid unhealthy deployments |
| health_check_ignore_transient_errors | boolean | If true, 429 (rate limit) and 408 (timeout) health check failures are ignored and do not affect routing or cooldown |
| enable_mcp_registry | boolean | If true, enables access to the centralized MCP server registry |
| enforce_rbac | boolean | If true, enables role-based access control (RBAC) for all proxy operations |
| forward_llm_provider_auth_headers | boolean | If true, forwards provider-specific auth headers to LLM API calls |
@ -396,6 +398,7 @@ router_settings:
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
| enable_health_check_routing | boolean | If true, enables health check-driven deployment filtering to avoid routing requests to unhealthy deployments |
| health_check_staleness_threshold | integer | Maximum age in seconds for cached health check results before marking deployments as stale |
| health_check_ignore_transient_errors | boolean | If true, 429 (rate limit) and 408 (timeout) health check failures are ignored and do not affect routing or cooldown |
### environment variables - Reference
@ -594,10 +597,13 @@ router_settings:
| LITELLM_MCP_TOOL_LISTING_TIMEOUT | Timeout in seconds for listing tools from an MCP server. Default is 30
| LITELLM_MCP_METADATA_TIMEOUT | HTTP client timeout in seconds for OAuth metadata fetching. Default is 10
| LITELLM_MCP_HEALTH_CHECK_TIMEOUT | Health check timeout in seconds for MCP servers. Default is 10
| LITELLM_MCP_STDIO_EXTRA_COMMANDS | Comma-separated extra command basenames allowed for MCP stdio transport beyond the built-in allowlist. Example: `my-mcp-bin`. Empty by default
| MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600
| MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200
| MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10
| MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS | Seconds to subtract from token expiry when computing cache TTL. Default is 60
| MCP_PER_USER_TOKEN_DEFAULT_TTL | Default TTL in seconds for per-user MCP OAuth tokens stored in Redis. Default is 43200 (12 hours)
| MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS | Seconds to subtract from per-user MCP OAuth token expiry when computing Redis TTL. Default is 60
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
@ -820,6 +826,7 @@ router_settings:
| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
| LITELLM_KEY_ROTATION_GRACE_PERIOD | Duration to keep old key valid after rotation (e.g. "24h", "2d"). Default is empty (immediate revoke). Used for scheduled rotations and as fallback when not specified in regenerate request.
| LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS | TTL in seconds for the distributed lock used by the key rotation job. Default is 600 (10 minutes).
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False`
| LITELLM_LOCAL_BLOG_POSTS | When set to `True`, uses the local bundled blog posts only, disabling remote fetching from GitHub. Default is `False`
@ -1028,6 +1035,7 @@ router_settings:
| SENDGRID_SENDER_EMAIL | Email address used as the sender in SendGrid email transactions
| SPEND_LOGS_URL | URL for retrieving spend logs
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000
| STALE_OBJECT_CLEANUP_BATCH_SIZE | Max number of stale managed objects updated per cleanup cycle. Default is 1000
| SSL_CERTIFICATE | Path to the SSL certificate file
| SSL_ECDH_CURVE | ECDH curve for SSL/TLS key exchange (e.g., 'X25519' to disable PQC).
| SSL_SECURITY_LEVEL | [BETA] Security level for SSL/TLS connections. E.g. `DEFAULT@SECLEVEL=1`

274
docs/my-website/docs/proxy/credential_routing.md Normal file
View file

@ -0,0 +1,274 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Per-Team/Project Credential Routing
Route the same model to different LLM provider endpoints (e.g. different Azure instances) based on which team or project makes the request.
## Overview
In multi-tenant deployments, different teams often need the same model name (e.g., `gpt-4`) to hit different provider endpoints — for example, separate Azure OpenAI instances per business unit for cost isolation, data residency, or rate limit separation.
**Credential routing** lets you configure this in team/project metadata using the existing [credentials table](./ui_credentials.md), without duplicating model definitions or creating separate model groups per team.
```
Hotel Team → gpt-4 → https://hotel-eastus.openai.azure.com/
Flight Team → gpt-4 → https://flight-centralus.openai.azure.com/
```
### Precedence Chain
When a request comes in, the system walks this precedence chain (first match wins):
1. **Clientside credentials** — `api_base`/`api_key` passed in the request body ([docs](./clientside_auth.md))
2. **Project model-specific** — override for this exact model in the project's `model_config`
3. **Project default** — `defaultconfig` in the project's `model_config`
4. **Team model-specific** — override for this exact model in the team's `model_config`
5. **Team default** — `defaultconfig` in the team's `model_config`
6. **Deployment default** — the model's `litellm_params` as configured in `config.yaml`
## Quick Start
### Step 1: Create Credentials
Store your Azure endpoint credentials in the credentials table. You can do this via the [UI](./ui_credentials.md) or API:
```bash showLineNumbers
# Create credential for Hotel team's Azure endpoint
curl -X POST 'http://0.0.0.0:4000/credentials' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"credential_name": "hotel-azure-eastus",
"credential_values": {
"api_base": "https://hotel-eastus.openai.azure.com/",
"api_key": "sk-azure-hotel-key-xxx"
}
}'
```
```bash showLineNumbers
# Create credential for Flight team's Azure endpoint
curl -X POST 'http://0.0.0.0:4000/credentials' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"credential_name": "flight-azure-centralus",
"credential_values": {
"api_base": "https://flight-centralus.openai.azure.com/",
"api_key": "sk-azure-flight-key-xxx"
}
}'
```
### Step 2: Set `model_config` on Teams
Add a `model_config` key to the team's metadata referencing the credential by name:
```bash showLineNumbers
# Hotel team — default Azure endpoint for all models
curl -X PATCH 'http://0.0.0.0:4000/team/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"team_id": "hotel-team-id",
"metadata": {
"model_config": {
"defaultconfig": {
"azure": {
"litellm_credentials": "hotel-azure-eastus"
}
}
}
}
}'
```
```bash showLineNumbers
# Flight team — default Azure endpoint for all models
curl -X PATCH 'http://0.0.0.0:4000/team/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"team_id": "flight-team-id",
"metadata": {
"model_config": {
"defaultconfig": {
"azure": {
"litellm_credentials": "flight-azure-centralus"
}
}
}
}
}'
```
### Step 3: Make Requests
Requests are automatically routed to the correct Azure endpoint based on the API key's team:
```bash showLineNumbers
# Request using Hotel team's API key → routes to hotel-eastus.openai.azure.com
curl http://localhost:4000/v1/chat/completions \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-hotel-team-key' \
-d '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}'
# Request using Flight team's API key → routes to flight-centralus.openai.azure.com
curl http://localhost:4000/v1/chat/completions \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-flight-team-key' \
-d '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}'
```
## Per-Model Overrides
You can set different credentials for specific models while keeping a default for everything else:
```bash showLineNumbers
curl -X PATCH 'http://0.0.0.0:4000/team/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"team_id": "hotel-team-id",
"metadata": {
"model_config": {
"defaultconfig": {
"azure": {
"litellm_credentials": "hotel-azure-eastus"
}
},
"gpt-4": {
"azure": {
"litellm_credentials": "hotel-azure-westus"
}
}
}
}
}'
```
With this config:
- `gpt-4` requests → `hotel-azure-westus` credential (model-specific)
- All other models → `hotel-azure-eastus` credential (default)
## Project-Level Overrides
Projects inherit their team's `model_config` but can override at the project level. Project overrides take precedence over team overrides.
```bash showLineNumbers
# Project overrides the team default for all models
curl -X PATCH 'http://0.0.0.0:4000/project/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"project_id": "hotel-rec-app-id",
"metadata": {
"model_config": {
"defaultconfig": {
"azure": {
"litellm_credentials": "hotel-rec-azure"
}
},
"gpt-4-vision": {
"azure": {
"litellm_credentials": "hotel-rec-vision"
}
}
}
}
}'
```
### Full Example: Hotel Team with Two Projects
**Setup:**
- **Hotel Team**: default `hotel-azure-eastus`, GPT-4 override to `hotel-azure-westus`
- **Hotel Rec App** (project): default `hotel-rec-azure`, GPT-4-Vision override to `hotel-rec-vision`
- **Hotel Review App** (project): no overrides — inherits team config
**Resolution:**
| Request | Resolved Credential | Why |
|---|---|---|
| Hotel Rec App → `gpt-4` | `hotel-rec-azure` | Project default (no project model-specific match for gpt-4) |
| Hotel Rec App → `gpt-4-vision` | `hotel-rec-vision` | Project model-specific |
| Hotel Review App → `gpt-3.5` | `hotel-azure-eastus` | Team default (no project config) |
| Hotel Review App → `gpt-4` | `hotel-azure-westus` | Team model-specific |
## `model_config` Schema
The `model_config` key is a JSON object in team/project `metadata`:
```json
{
"model_config": {
"defaultconfig": {
"<provider>": {
"litellm_credentials": "<credential-name>"
}
},
"<model-name>": {
"<provider>": {
"litellm_credentials": "<credential-name>"
}
}
}
}
```
| Field | Description |
|---|---|
| `defaultconfig` | Fallback credential for any model not explicitly listed |
| `<model-name>` | Model-specific override — must match the LiteLLM model group name |
| `<provider>` | Provider key (e.g. `azure`, `openai`, `bedrock`). When the model name includes a provider prefix (e.g. `azure/gpt-4`), the system prefers the matching provider key |
| `litellm_credentials` | Name of a credential in the [credentials table](./ui_credentials.md) |
### Credential Values
The referenced credential can contain any combination of:
| Key | Description |
|---|---|
| `api_base` | Provider endpoint URL |
| `api_key` | API key for the provider |
| `api_version` | API version (e.g. for Azure) |
Only keys present in the credential are applied. Keys already in the request (e.g. clientside `api_version`) are never overwritten.
## Enabling the Feature
This feature is **disabled by default** and must be explicitly enabled. To enable it:
<Tabs>
<TabItem value="config" label="config.yaml">
```yaml
litellm_settings:
enable_model_config_credential_overrides: true
```
</TabItem>
<TabItem value="env" label="Environment Variable">
```bash
export LITELLM_ENABLE_MODEL_CONFIG_CREDENTIAL_OVERRIDES=true
```
</TabItem>
</Tabs>
:::info
The feature flag must be enabled before `model_config` entries in team/project metadata take effect. Without it, credential routing is completely inert — no metadata is read, no credentials are resolved.
:::
## Related Documentation
- [Adding LLM Credentials](./ui_credentials.md) — Create and manage reusable credentials
- [Project Management](./project_management.md) — Project hierarchy and API
- [Team Budgets](./team_budgets.md) — Team-level budget management
- [Clientside LLM Credentials](./clientside_auth.md) — Passing credentials in the request body
- [Credential Usage Tracking](./credential_usage_tracking.md) — Track spend by credential

38
docs/my-website/docs/proxy/deploy.md
View file

@ -65,7 +65,43 @@ docker compose up
</TabItem>
</Tabs>
### Docker Run
### Verify Docker image signatures
All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
**Verify using a release tag (convenience):**
Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
Expected output:
```
The following checks were performed on each of these signatures:
- The cosign claims were validated
- The signatures were verified against the specified public key
```
Learn more about LiteLLM's release signing in the [CI/CD v2 announcement](https://docs.litellm.ai/blog/ci-cd-v2-improvements#verify-docker-image-signatures). For a complete guide covering all image variants, CI/CD enforcement, and deployment best practices, see the [Docker Image Security Guide](./docker_image_security.md).
### Docker Run
#### Step 1. CREATE config.yaml

189
docs/my-website/docs/proxy/docker_image_security.md Normal file
View file

@ -0,0 +1,189 @@
# Docker Image Security Guide
LiteLLM signs every Docker image published to GHCR with [cosign](https://docs.sigstore.dev/cosign/overview/) starting from **v1.83.0**. This page covers how to verify signatures, enforce verification in CI/CD, and follow recommended deployment patterns.
## Signed images
All image variants published to `ghcr.io/berriai/` are signed with the same cosign key:
| Image | Description |
|---|---|
| `ghcr.io/berriai/litellm` | Core proxy |
| `ghcr.io/berriai/litellm-database` | Proxy with Postgres dependencies |
| `ghcr.io/berriai/litellm-non_root` | Non-root variant |
| `ghcr.io/berriai/litellm-spend_logs` | Spend-logs sidecar |
The signing key was introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0) and the public key is checked into the repository at [`cosign.pub`](https://github.com/BerriAI/litellm/blob/main/cosign.pub).
:::info Enterprise images
Enterprise images (`litellm-ee`) follow the same signing process. Contact [support@berri.ai](mailto:support@berri.ai) to confirm coverage for your specific enterprise image tag.
:::
## Verify image signatures
Install cosign following the [official instructions](https://docs.sigstore.dev/cosign/system_config/installation/).
### Verify with the pinned commit hash (recommended)
A commit hash is cryptographically immutable, making this the strongest verification method:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm:v1.83.0-stable
```
Replace the image reference with any signed variant:
```bash
# litellm-database
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm-database:v1.83.0-stable
# litellm-non_root
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm-non_root:v1.83.0-stable
```
### Verify with a release tag (convenience)
Tags are protected in this repository and resolve to the same key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/v1.83.0-stable/cosign.pub \
ghcr.io/berriai/litellm-database:v1.83.0-stable
```
### Expected output
```
The following checks were performed on each of these signatures:
- The cosign claims were validated
- The signatures were verified against the specified public key
```
## Enforce verification in CI/CD
### Kubernetes — Sigstore Policy Controller
The [Sigstore Policy Controller](https://docs.sigstore.dev/policy-controller/overview/) rejects pods whose images fail cosign verification.
1. Install the controller:
```bash
helm repo add sigstore https://sigstore.github.io/helm-charts
helm install policy-controller sigstore/policy-controller \
-n cosign-system --create-namespace
```
2. Create a `ClusterImagePolicy` with the LiteLLM public key:
```yaml
apiVersion: policy.sigstore.dev/v1beta1
kind: ClusterImagePolicy
metadata:
name: litellm-signed-images
spec:
images:
- glob: "ghcr.io/berriai/litellm*"
authorities:
- key:
data: |
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
-----END PUBLIC KEY-----
```
3. Label the namespace to enable enforcement:
```bash
kubectl label namespace litellm policy.sigstore.dev/include=true
```
Any pod in that namespace using an unsigned `ghcr.io/berriai/litellm*` image will be rejected at admission.
### GCP — Binary Authorization
[Binary Authorization](https://cloud.google.com/binary-authorization/docs) can enforce cosign signatures on Cloud Run and GKE.
1. Create a cosign-based attestor using the LiteLLM public key:
```bash
# Import the public key into a Cloud KMS keyring or use a PGP/PKIX attestor.
# See: https://cloud.google.com/binary-authorization/docs/creating-attestors-console
```
2. Configure a Binary Authorization policy that requires the attestor for `ghcr.io/berriai/litellm*` images.
3. Enable the policy on your Cloud Run service or GKE cluster.
Refer to the [GCP Binary Authorization docs](https://cloud.google.com/binary-authorization/docs/setting-up) for full setup steps.
### AWS — ECS / ECR
AWS does not natively verify cosign signatures at deploy time. Common approaches:
- **CI/CD gate**: Run `cosign verify` in your deployment pipeline before pushing to ECR or updating the ECS task definition. Fail the pipeline if verification fails.
- **OPA/Gatekeeper on EKS**: If running on EKS, use the Sigstore Policy Controller (same as the Kubernetes approach above).
### GitHub Actions gate
Add a verification step before any deployment job:
```yaml
- name: Verify LiteLLM image signature
run: |
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm-database:${{ env.LITELLM_VERSION }}
```
## Recommended deployment patterns
### Pin by digest
Digest pinning guarantees the exact image content regardless of tag mutations:
```yaml
image: ghcr.io/berriai/litellm-database@sha256:<digest>
```
Get the digest after pulling:
```bash
docker inspect --format='{{index .RepoDigests 0}}' \
ghcr.io/berriai/litellm-database:v1.83.0-stable
```
Cosign verification works with digests too:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm-database@sha256:<digest>
```
### Use stable release tags
If digest pinning is too rigid for your workflow, use `-stable` release tags (e.g. `v1.83.0-stable`). These are immutable release tags that will not be overwritten.
Avoid `main-latest` or `main-stable` in production — these rolling tags point to the most recent build and can change between deployments.
### Safe upgrade checklist
1. **Verify the new image** — run `cosign verify` against the new release tag or digest.
2. **Test in staging** — deploy the verified image to a non-production environment.
3. **Update your pinned reference** — change the digest or tag in your deployment manifest.
4. **Deploy to production** — roll out using your standard deployment process.
5. **Monitor `/health`** — confirm the proxy is healthy after the upgrade.
## Further reading
- [CI/CD v2 announcement](https://docs.litellm.ai/blog/ci-cd-v2-improvements) — background on LiteLLM's signing infrastructure
- [Docker deployment guide](./deploy.md) — full Docker, Helm, and Terraform setup
- [cosign documentation](https://docs.sigstore.dev/cosign/overview/) — cosign usage and key management
- [Sigstore Policy Controller](https://docs.sigstore.dev/policy-controller/overview/) — Kubernetes admission control

4
docs/my-website/docs/proxy/guardrails/guardrail_policies.md
View file

@ -311,7 +311,7 @@ Response:
## Policy Flow Builder
For conditional execution (e.g., run a second guardrail only if the first fails), use the [Policy Flow Builder](./policy_flow_builder) to define pipelines with per-step pass/fail actions.
For conditional execution (e.g., run a second guardrail only if the first fails), use the [Policy Flow Builder](./policy_flow_builder) to define pipelines with per-step **pass**, **fail**, and optional **error** actions (`on_pass`, `on_fail`, `on_error`).
## Config Reference
@ -337,7 +337,7 @@ policies:
| `guardrails.add` | `list[string]` | Guardrails to enable. |
| `guardrails.remove` | `list[string]` | Guardrails to disable (useful with inheritance). |
| `condition.model` | `string` or `list[string]` | Optional. Only apply when model matches. Supports regex. |
| `pipeline` | `object` | Optional. Ordered guardrail execution with per-step actions. See [Policy Flow Builder](./policy_flow_builder). |
| `pipeline` | `object` | Optional. Ordered guardrail execution with per-step actions (`on_pass`, `on_fail`, optional `on_error`). See [Policy Flow Builder](./policy_flow_builder). |
### `policy_attachments`

65
docs/my-website/docs/proxy/guardrails/policy_flow_builder.md
View file

@ -1,8 +1,8 @@
# Policy Flow Builder
The Policy Flow Builder lets you design guardrail pipelines with **conditional execution**. Instead of running guardrails independently, you chain them into ordered steps and control what happens when each guardrail passes or fails.
The Policy Flow Builder lets you design guardrail pipelines with **conditional execution**. Instead of running guardrails independently, you chain them into ordered steps and control what happens when each guardrail **passes**, **fails a policy check** (content intervention), or hits a **technical error** (e.g. timeout, unreachable provider, missing guardrail).
Two powerful patterns it enables: **guardrail fallbacks** (try a different guardrail when one fails) and **retrying the same guardrail** (run the same guardrail again if it fails, e.g. to handle transient errors).
Two powerful patterns it enables: **guardrail fallbacks** (try a different guardrail when one fails) and **retrying the same guardrail** (run the same guardrail again if it fails, e.g. to handle transient errors). With **`on_error`**, you can treat **technical** failures differently from **policy** failures—for example, fall back to another provider when the primary API errors, while still blocking on flagged content.
## When to use the Flow Builder
@ -19,6 +19,7 @@ Use the Flow Builder when you need:
- **Custom responses** — return a specific message when a guardrail fails instead of a generic block
- **Data chaining** — pass modified data (e.g., PII-masked content) from one step to the next
- **Fine-grained control** — different actions on pass vs. fail per step
- **Technical-error routing** — set `on_error` separately from `on_fail` so outages or timeouts can **allow**, **block**, **go to the next step**, or return a **custom response** without conflating them with content violations
## Concepts
@ -29,24 +30,37 @@ A pipeline has:
- **Mode**: `pre_call` (before the LLM) or `post_call` (after the LLM)
- **Steps**: Ordered list of guardrail steps
### Outcomes: pass, fail, and error
Each step run produces one of three outcomes:
| Outcome | Meaning | Typical cause |
|--------|---------|----------------|
| **pass** | Guardrail completed without blocking | Content allowed, or data was modified and returned |
| **fail** | Policy intervention | Guardrail raised an intervention (e.g. flagged content, blocked request) |
| **error** | Technical failure | Timeouts, network errors, guardrail not registered, or other non-intervention exceptions |
`on_pass` and `on_fail` apply to **pass** and **fail** respectively. **`on_error`** applies only to **error**. If `on_error` is omitted, the pipeline uses **`on_fail`** for error outcomes (backward compatible).
### Step actions
Each step defines what happens when the guardrail **passes** and when it **fails**:
For each step you choose an action for **pass**, **fail**, and optionally **error**. Allowed values are: `next`, `allow`, `block`, `modify_response`.
| Action | Description |
|--------|-------------|
| **Next Step** | Continue to the next guardrail in the pipeline |
| **Allow** | Stop the pipeline and allow the request to proceed |
| **Block** | Stop the pipeline and block the request |
| **Custom Response** | Return a custom message instead of the default block |
| **Next Step** (`next`) | Continue to the next guardrail in the pipeline |
| **Allow** (`allow`) | Stop the pipeline and allow the request to proceed |
| **Block** (`block`) | Stop the pipeline and block the request |
| **Custom Response** (`modify_response`) | Return a custom message instead of the default block |
### Step options
| Field | Type | Description |
|-------|------|--------------|
| `guardrail` | `string` | Name of the guardrail to run |
| `on_pass` | `string` | Action when guardrail passes: `next`, `allow`, `block`, `modify_response` |
| `on_fail` | `string` | Action when guardrail fails: `next`, `allow`, `block`, `modify_response` |
| `on_pass` | `string` | Action when outcome is **pass**: `next`, `allow`, `block`, `modify_response` |
| `on_fail` | `string` | Action when outcome is **fail** (policy intervention): `next`, `allow`, `block`, `modify_response` |
| `on_error` | `string` (optional) | Action when outcome is **error** (technical). If omitted, **error** uses `on_fail`. |
| `pass_data` | `boolean` | Forward modified request data (e.g., PII-masked) to the next step |
| `modify_response_message` | `string` | Custom message when using `modify_response` action |
@ -57,7 +71,7 @@ Each step defines what happens when the guardrail **passes** and when it **fails
3. Select **Flow Builder** (instead of the simple form)
4. Design your flow:
- **Trigger** — Incoming LLM request (runs when the policy matches)
- **Steps** — Add guardrails, set ON PASS and ON FAIL actions per step
- **Steps** — Add guardrails, set **ON PASS**, **ON FAIL**, and **ON ERROR** actions per step (ON ERROR is optional; when unset, errors follow ON FAIL)
- **End** — Request proceeds to the LLM
5. Use the **+** between steps to insert new steps
6. Use the **Test** panel to run sample messages through the pipeline before saving
@ -151,6 +165,37 @@ policies:
First attempt passes → allow. First attempt fails → retry the same guardrail; second pass → allow, second fail → block.
## Technical errors vs policy failures (`on_error`)
Use **`on_error`** when you want different behavior for **API/infra problems** than for **content policy** violations.
- **`on_fail`** — Runs when the guardrail **intervenes** (e.g. toxic content, PII detected).
- **`on_error`** — Runs when the step ends in **error** (timeout, connection failure, guardrail not loaded, etc.). If you omit `on_error`, **error** outcomes use **`on_fail`**.
Example: block on bad content, but if the primary scanner is down, fall back to a second guardrail instead of blocking every request:
```yaml
policies:
error-fallback-policy:
guardrails:
add:
- primary_scanner
- backup_scanner
pipeline:
mode: pre_call
steps:
- guardrail: primary_scanner
on_pass: allow
on_fail: block
on_error: next
- guardrail: backup_scanner
on_pass: allow
on_fail: block
on_error: allow
```
If `primary_scanner` errors → run `backup_scanner`. If `backup_scanner` errors → allow the request (set `on_error` to `block` if you prefer fail-closed).
## Example: Custom response on fail
Return a branded message instead of a generic block:

81
docs/my-website/docs/proxy/health.md
View file

@ -316,86 +316,9 @@ general_settings:
## Health Check Driven Routing
By default, background health checks are observability-only — they populate the `/health` endpoint but don't affect routing. Unhealthy deployments still receive traffic until request failures trigger cooldown.
Route traffic away from unhealthy deployments proactively — before user requests hit them. Supports per-error-type failure thresholds, transient error suppression, and automatic safety nets.
With `enable_health_check_routing: true`, the router **excludes deployments that failed their last background health check** before selecting a candidate. This gives you proactive failover instead of reactive cooldown.
### How it works
1. Background health checks run on their configured interval
2. After each cycle, every deployment is marked healthy or unhealthy
3. On each incoming request, the router filters out unhealthy deployments **before** cooldown filtering and load balancing
4. If all deployments are unhealthy, the filter is bypassed (safety net — never causes a total outage)
5. If health state is stale (older than `health_check_staleness_threshold`), it is ignored
### Quick start
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY_SECONDARY
general_settings:
background_health_checks: true
health_check_interval: 60
enable_health_check_routing: true
```
### Configuration
| Setting | Where | Default | Description |
|---------|-------|---------|-------------|
| `enable_health_check_routing` | `general_settings` | `false` | Enable/disable health-check-driven routing |
| `health_check_staleness_threshold` | `general_settings` | `health_check_interval * 2` | Seconds before health state is considered stale and ignored |
| `background_health_checks` | `general_settings` | `false` | Must be `true` for health check routing to work |
| `health_check_interval` | `general_settings` | `300` | Seconds between health check cycles |
### Interaction with cooldown
Health check filtering and cooldown are **additive**. A deployment can be excluded by either mechanism:
- **Health check filter** — proactive, runs on the configured interval, excludes deployments that failed the last check
- **Cooldown** — reactive, triggered by request failures, excludes deployments for a short TTL
This means request failures still provide fast detection between health check intervals.
### Staleness
If a health check result is older than `health_check_staleness_threshold`, it is ignored and the deployment is treated as eligible. This prevents stale data from permanently excluding a deployment if the health check loop stops or slows down.
The default staleness threshold is `health_check_interval * 2`. For a 60s interval, health state expires after 120s.
### Example: custom staleness
```yaml
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
health_check_staleness_threshold: 90 # ignore health state older than 90s
```
### Debugging
Run the proxy with `--detailed_debug` and look for:
```
health_check_routing_state_updated healthy=3 unhealthy=1
```
This is logged after each health check cycle when routing state is written.
If the safety net triggers (all deployments unhealthy), you'll see:
```
All deployments marked unhealthy by health checks, bypassing health filter
```
See the full guide: [Health Check Driven Routing](./health_check_routing.md)
## Health Check Timeout

340
docs/my-website/docs/proxy/health_check_routing.md Normal file
View file

@ -0,0 +1,340 @@
# Health Check Driven Routing
Route traffic away from unhealthy deployments before users hit errors. Background health checks run on a configurable interval, and any deployment that fails gets removed from the routing pool proactively, not after a user request already failed.
## Architecture
<svg viewBox="0 0 860 600" xmlns="http://www.w3.org/2000/svg" style={{maxWidth: '100%', fontFamily: 'system-ui, sans-serif'}}>
{/* Background */}
<rect width="860" height="600" fill="#f8fafc" rx="12"/>
{/* LEFT PANEL: Background health check loop */}
<rect x="20" y="20" width="240" height="560" fill="#eff6ff" rx="10" stroke="#bfdbfe" strokeWidth="1.5"/>
<text x="140" y="48" textAnchor="middle" fill="#1d4ed8" fontSize="13" fontWeight="600">Background Loop</text>
<text x="140" y="64" textAnchor="middle" fill="#3b82f6" fontSize="11">every health_check_interval seconds</text>
{/* Deployment A */}
<rect x="40" y="82" width="200" height="50" fill="white" rx="8" stroke="#93c5fd" strokeWidth="1.5"/>
<text x="140" y="102" textAnchor="middle" fill="#1e40af" fontSize="12" fontWeight="500">Deployment A</text>
<text x="140" y="120" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 200 ✓</text>
{/* Deployment B */}
<rect x="40" y="148" width="200" height="50" fill="white" rx="8" stroke="#fca5a5" strokeWidth="1.5"/>
<text x="140" y="168" textAnchor="middle" fill="#991b1b" fontSize="12" fontWeight="500">Deployment B</text>
<text x="140" y="186" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 401 ✗</text>
{/* Deployment C */}
<rect x="40" y="214" width="200" height="50" fill="white" rx="8" stroke="#fde68a" strokeWidth="1.5"/>
<text x="140" y="234" textAnchor="middle" fill="#92400e" fontSize="12" fontWeight="500">Deployment C</text>
<text x="140" y="252" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 429 ⚡</text>
{/* ignore_transient box */}
<rect x="40" y="282" width="200" height="68" fill="#fefce8" rx="8" stroke="#fde047" strokeWidth="1.5"/>
<text x="140" y="302" textAnchor="middle" fill="#713f12" fontSize="11" fontWeight="600">ignore_transient_errors: true</text>
<text x="140" y="320" textAnchor="middle" fill="#92400e" fontSize="11">429 / 408 → ignored</text>
<text x="140" y="338" textAnchor="middle" fill="#92400e" fontSize="11">not written to cache</text>
{/* allowed_fails_policy box */}
<rect x="40" y="368" width="200" height="84" fill="#f0fdf4" rx="8" stroke="#86efac" strokeWidth="1.5"/>
<text x="140" y="388" textAnchor="middle" fill="#166534" fontSize="11" fontWeight="600">allowed_fails_policy</text>
<text x="140" y="406" textAnchor="middle" fill="#15803d" fontSize="11">401 → increment counter</text>
<text x="140" y="424" textAnchor="middle" fill="#15803d" fontSize="11">counter &gt; threshold</text>
<text x="140" y="442" textAnchor="middle" fill="#15803d" fontSize="11">→ cooldown triggered</text>
{/* CENTER PANEL: Shared State */}
<rect x="300" y="20" width="220" height="560" fill="#f5f3ff" rx="10" stroke="#c4b5fd" strokeWidth="1.5"/>
<text x="410" y="48" textAnchor="middle" fill="#6d28d9" fontSize="13" fontWeight="600">Shared State</text>
{/* Health State Cache */}
<rect x="320" y="62" width="180" height="116" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
<text x="410" y="84" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">DeploymentHealthCache</text>
<text x="410" y="104" textAnchor="middle" fill="#64748b" fontSize="11">A → healthy ✓</text>
<text x="410" y="122" textAnchor="middle" fill="#64748b" fontSize="11">B → unhealthy ✗</text>
<text x="410" y="140" textAnchor="middle" fill="#64748b" fontSize="11">C → not written (ignored)</text>
<text x="410" y="164" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: staleness_threshold × 1.5</text>
{/* Cooldown Cache */}
<rect x="320" y="196" width="180" height="104" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
<text x="410" y="218" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">Cooldown Cache</text>
<text x="410" y="238" textAnchor="middle" fill="#64748b" fontSize="11">B → cooling down</text>
<text x="410" y="256" textAnchor="middle" fill="#64748b" fontSize="11">(after policy threshold)</text>
<text x="410" y="278" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: cooldown_time</text>
{/* failed_calls counter */}
<rect x="320" y="318" width="180" height="90" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
<text x="410" y="340" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">failed_calls counter</text>
<text x="410" y="360" textAnchor="middle" fill="#64748b" fontSize="11">B: 2 / AuthAllowedFails: 1</text>
<text x="410" y="378" textAnchor="middle" fill="#64748b" fontSize="11">→ threshold exceeded</text>
<text x="410" y="398" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: cooldown_time (must &gt; interval)</text>
{/* RIGHT PANEL: Request path */}
<rect x="560" y="20" width="280" height="560" fill="#fff7ed" rx="10" stroke="#fed7aa" strokeWidth="1.5"/>
<text x="700" y="48" textAnchor="middle" fill="#c2410c" fontSize="13" fontWeight="600">Request Path</text>
{/* Incoming request */}
<rect x="580" y="62" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="85" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="500">Incoming request</text>
{/* All deployments */}
<rect x="580" y="120" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="143" textAnchor="middle" fill="#9a3412" fontSize="12">All deployments [A, B, C]</text>
<line x1="700" y1="100" x2="700" y2="120" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Health check filter */}
<rect x="580" y="178" width="240" height="62" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="200" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">① Health Check Filter</text>
<text x="700" y="218" textAnchor="middle" fill="#64748b" fontSize="11">if policy set → bypass</text>
<text x="700" y="234" textAnchor="middle" fill="#64748b" fontSize="11">else → remove unhealthy</text>
<line x1="700" y1="158" x2="700" y2="178" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Cooldown filter */}
<rect x="580" y="262" width="240" height="50" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="284" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">② Cooldown Filter</text>
<text x="700" y="302" textAnchor="middle" fill="#64748b" fontSize="11">remove deployments in cooldown</text>
<line x1="700" y1="240" x2="700" y2="262" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Safety net */}
<rect x="580" y="334" width="240" height="52" fill="#fef9c3" rx="7" stroke="#fbbf24" strokeWidth="1.5"/>
<text x="700" y="356" textAnchor="middle" fill="#713f12" fontSize="12" fontWeight="600">Safety Net</text>
<text x="700" y="376" textAnchor="middle" fill="#713f12" fontSize="11">if all removed → return all</text>
<line x1="700" y1="312" x2="700" y2="334" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Load balancer */}
<rect x="580" y="408" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="431" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">③ Load Balancer</text>
<line x1="700" y1="386" x2="700" y2="408" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Selected deployment */}
<rect x="580" y="468" width="240" height="38" fill="#dcfce7" rx="7" stroke="#4ade80" strokeWidth="1.5"/>
<text x="700" y="491" textAnchor="middle" fill="#14532d" fontSize="12" fontWeight="600">Selected: Deployment A ✓</text>
<line x1="700" y1="446" x2="700" y2="468" stroke="#4ade80" strokeWidth="1.5" markerEnd="url(#arrow-green)"/>
{/* ARROWS: left → center */}
<line x1="240" y1="107" x2="320" y2="110" stroke="#3b82f6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-blue)"/>
<line x1="240" y1="173" x2="320" y2="240" stroke="#ef4444" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-red)"/>
<line x1="240" y1="173" x2="320" y2="348" stroke="#ef4444" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-red)"/>
<line x1="240" y1="316" x2="320" y2="130" stroke="#eab308" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-yellow)"/>
{/* ARROWS: center → right */}
<line x1="500" y1="120" x2="580" y2="190" stroke="#8b5cf6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-purple)"/>
<line x1="500" y1="248" x2="580" y2="274" stroke="#8b5cf6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-purple)"/>
{/* Arrow markers */}
<defs>
<marker id="arrow-orange" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#fb923c"/>
</marker>
<marker id="arrow-blue" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#3b82f6"/>
</marker>
<marker id="arrow-red" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#ef4444"/>
</marker>
<marker id="arrow-yellow" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#eab308"/>
</marker>
<marker id="arrow-purple" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#8b5cf6"/>
</marker>
<marker id="arrow-green" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#4ade80"/>
</marker>
</defs>
</svg>
## What problem does this solve?
By default, LiteLLM routes traffic to all deployments and only stops sending to a broken one after it has already failed a user request. The cooldown system is reactive.
Health check driven routing makes this **proactive**: a background loop pings every deployment on a configurable interval. If a deployment fails its health check, it gets removed from the routing pool immediately, before a user request lands on it.
When you also set `allowed_fails_policy`, you control exactly how many health check failures of each error type (auth errors, rate limits, timeouts) are needed before a deployment enters cooldown. This avoids false positives from transient noise.
## Setup
### Step 1: Enable background health checks
Background health checks are off by default. Turn them on in `general_settings`:
```yaml
general_settings:
background_health_checks: true
health_check_interval: 60 # seconds between each full check cycle
```
### Step 2: Enable health check routing
```yaml
general_settings:
background_health_checks: true
health_check_interval: 60
enable_health_check_routing: true # ← route away from unhealthy deployments
```
At this point, any deployment that fails its health check is immediately excluded from routing until the next check cycle clears it.
### Step 3: Add a policy to control how many failures trigger cooldown
Without a policy, the first health check failure marks a deployment as unhealthy. If you want more tolerance (e.g., only act after 2 consecutive auth failures), use `allowed_fails_policy`:
```yaml
model_list:
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-5
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-5
api_key: os.environ/ANTHROPIC_API_KEY_SECONDARY
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
router_settings:
cooldown_time: 60 # how long a deployment stays in cooldown
allowed_fails_policy:
AuthenticationErrorAllowedFails: 1 # cooldown after 2nd auth failure
TimeoutErrorAllowedFails: 3 # cooldown after 4th timeout
```
When `allowed_fails_policy` is set, the binary health check filter is bypassed. Only the cooldown system controls routing exclusion, and it only fires after your configured threshold is crossed.
### Step 4 (optional): Ignore transient errors
429 (rate limit) and 408 (timeout) from a health check usually mean the deployment is temporarily overloaded, not broken. To prevent these from affecting routing at all:
```yaml
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
health_check_ignore_transient_errors: true # 429 and 408 never affect routing
```
With this on, only hard failures (401, 404, 5xx) from health checks contribute to cooldown.
## Full example
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY_SECONDARY
- model_name: gpt-4o
litellm_params:
model: azure/gpt-4o
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
health_check_ignore_transient_errors: true
router_settings:
cooldown_time: 60
allowed_fails_policy:
AuthenticationErrorAllowedFails: 0 # cooldown immediately on auth failure
TimeoutErrorAllowedFails: 2 # cooldown after 3 timeouts
RateLimitErrorAllowedFails: 5 # cooldown after 6 rate limits (if not ignoring transients)
```
## Configuration reference
| Setting | Where | Default | Description |
|---|---|---|---|
| `enable_health_check_routing` | `general_settings` | `false` | Route away from deployments that fail health checks |
| `background_health_checks` | `general_settings` | `false` | Must be `true` for health check routing to work |
| `health_check_interval` | `general_settings` | `300` | Seconds between full health check cycles |
| `health_check_staleness_threshold` | `general_settings` | `interval x 2` | Seconds before cached health state is ignored |
| `health_check_ignore_transient_errors` | `general_settings` | `false` | Ignore 429 and 408 from health checks; these never affect routing |
| `cooldown_time` | `router_settings` | `5` | Seconds a deployment stays in cooldown after threshold is crossed |
| `allowed_fails_policy` | `router_settings` | `null` | Per-error-type failure thresholds before cooldown (see below) |
### `allowed_fails_policy` fields
| Field | Error type | HTTP status |
|---|---|---|
| `AuthenticationErrorAllowedFails` | Bad API key | 401 |
| `TimeoutErrorAllowedFails` | Request timeout | 408 |
| `RateLimitErrorAllowedFails` | Rate limit exceeded | 429 |
| `BadRequestErrorAllowedFails` | Malformed request | 400 |
| `ContentPolicyViolationErrorAllowedFails` | Content filtered | 400 |
The value is the number of failures **tolerated** before cooldown. `0` means cooldown on the first failure. `2` means cooldown on the third.
## Things to keep in mind
- **Counter TTL must be longer than the health check interval.** `allowed_fails_policy` works by incrementing a `failed_calls` counter per deployment. That counter expires after `cooldown_time` seconds. If `cooldown_time` is shorter than `health_check_interval`, the counter resets between every check cycle and failures never accumulate. Set `cooldown_time` greater than `health_check_interval` when using `allowed_fails_policy`.
```yaml
router_settings:
cooldown_time: 60 # must be > health_check_interval (30s here)
general_settings:
health_check_interval: 30
```
- **`AllowedFails: N` means cooldown on the (N+1)th failure.** The counter check is `updated_fails > allowed_fails`, so `0` triggers on the 1st failure, `1` on the 2nd, `2` on the 3rd.
| `AllowedFails` | Cooldown triggers after |
|---|---|
| `0` | 1st failure |
| `1` | 2nd failure |
| `2` | 3rd failure |
- **Without `allowed_fails_policy`, the first failure is enough.** The first failed health check immediately excludes the deployment from routing. Use `allowed_fails_policy` when you want tolerance for flaky checks.
- **If all deployments are unhealthy, the filter is bypassed.** Traffic keeps flowing rather than returning no deployment at all. Requests will fail, but the router keeps trying.
- **Health check failures and request failures share the same counters.** When `allowed_fails_policy` is set, both sources increment the same `failed_calls` counter. A deployment at 1 health check failure that then receives 1 failing request will hit the threshold for `AllowedFails: 1` and enter cooldown.
## Debugging
Run the proxy with `--detailed_debug` and look for these log lines:
After each health check cycle (written at DEBUG level):
```
health_check_routing_state_updated healthy=2 unhealthy=1
```
When a health check failure increments the counter and triggers cooldown (DEBUG level):
```
checks 'should_run_cooldown_logic'
Attempting to add <deployment_id> to cooldown list
```
When safety net fires because all deployments are in cooldown:
```
All deployments in cooldown via health-check routing, bypassing cooldown filter
```
When safety net fires because all deployments are unhealthy (binary filter, no `allowed_fails_policy`):
```
All deployments marked unhealthy by health checks, bypassing health filter
```

24
docs/my-website/docs/proxy/oauth2.md
View file

@ -61,3 +61,27 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
Start the LiteLLM Proxy with [`--detailed_debug` mode and you should see more verbose logs](cli.md#detailed_debug)
## Using OAuth2 + JWT Together
LiteLLM supports two OAuth2 + JWT modes:
1. **Global OAuth2 mode** (`enable_oauth2_auth: true`)
OAuth2 auth is enabled on LLM + info routes.
2. **Selective JWT override mode** (`enable_oauth2_auth: false`)
Only JWT-shaped tokens that match `litellm_jwtauth.routing_overrides` are routed to OAuth2 on LLM + info routes.
For selective routing (OAuth2 only for specific JWTs), configure:
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: false
litellm_jwtauth:
routing_overrides:
- iss: "machine-issuer.example.com"
client_id: "MID_LITELLM"
path: "oauth2"
```
For full `routing_overrides` behavior and list-based selectors, see [`/proxy/token_auth`](./token_auth.md#route-jwt-shaped-machine-tokens-to-oauth2).

25
docs/my-website/docs/proxy/self_serve.md
View file

@ -358,10 +358,15 @@ When you connect litellm to your SSO provider, litellm can auto-create teams. Us
```yaml showLineNumbers title="Default Params for new teams"
litellm_settings:
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
max_budget: 100 # Optional[float], optional): $100 budget for the team
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
max_budget: 100 # Optional[float]: $100 budget for the team
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
tpm_limit: 100000 # Optional[int]: tokens per minute limit
rpm_limit: 1000 # Optional[int]: requests per minute limit
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
- "/team/daily/activity" # Allow members to view team usage
- "/key/generate" # Allow members to generate API keys
```
@ -390,10 +395,14 @@ litellm_settings:
max_budget_in_team: 100 # Optional[float], optional): $100 budget for the team. Defaults to None.
user_role: "user" # Optional[str], optional): "user" or "admin". Defaults to "user"
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
max_budget: 100 # Optional[float], optional): $100 budget for the team
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
max_budget: 100 # Optional[float]: $100 budget for the team
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
tpm_limit: 100000 # Optional[int]: tokens per minute limit
rpm_limit: 1000 # Optional[int]: requests per minute limit
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
- "/team/daily/activity"
upperbound_key_generate_params: # Upperbound for /key/generate requests when self-serve flow is on

2
docs/my-website/docs/proxy/team_model_add.md
View file

@ -26,7 +26,7 @@ curl -L -X POST 'http://0.0.0.0:4000/model/new' \
"model": "openai/gpt-4o",
"custom_llm_provider": "openai",
"api_key": "******ccb07",
"api_base": "https://my-endpoint-sweden-berri992.openai.azure.com",
"api_base": "https://my-azure-endpoint.openai.azure.com",
"api_version": "2023-12-01-preview"
},
"model_info": {

43
docs/my-website/docs/proxy/token_auth.md
View file

@ -790,6 +790,49 @@ litellm_jwtauth:
user_roles_jwt_field: "resource_access.your-client.roles"
```
## Route JWT-Shaped Machine Tokens to OAuth2
Use this when:
- `enable_jwt_auth: true` for standard JWT validation
- machine tokens are JWT-shaped and should be routed to OAuth2 based on claims
`routing_overrides` supports two operating modes:
- **Selective mode**: set `enable_oauth2_auth: false` to send only matching JWTs to OAuth2 on LLM + info routes
- **Global mode**: set `enable_oauth2_auth: true` to also enable OAuth2 on LLM + info routes
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: false
litellm_jwtauth:
user_id_jwt_field: "sub"
routing_overrides:
- iss: "machine-issuer.example.com"
client_id: "MID_LITELLM"
path: "oauth2"
```
### Matching behavior
- A rule matches when all configured selectors match token claims
- Supported selectors: `iss` (required), `client_id` (optional), `aud` (optional)
- Selector values support both string and list forms
- If no rule matches, LiteLLM continues with standard JWT validation
### List-based override example
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: false
litellm_jwtauth:
routing_overrides:
- iss: ["machine-issuer.example.com", "backup-issuer.example.com"]
client_id: ["MID_LITELLM", "MID_BACKUP"]
aud: ["api://litellm", "api://fallback"]
path: "oauth2"
```
## [BETA] Control Access with OIDC Roles
Allow JWT tokens with supported roles to access the proxy.

2
docs/my-website/docs/realtime.md
View file

@ -82,7 +82,7 @@ Run this script using node - `node test.js`
const WebSocket = require("ws");
const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio";
// const url = "wss://my-endpoint-sweden-berri992.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview";
// const url = "wss://my-azure-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview";
const ws = new WebSocket(url, {
headers: {
"api-key": `sk-1234`,

10
docs/my-website/docs/tutorials/msft_sso.md
View file

@ -123,10 +123,12 @@ Navigate to your litellm config file and set the following params
```yaml showLineNumbers title="litellm config with default_team_params"
litellm_settings:
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
max_budget: 100 # Optional[float], optional): $100 budget for the team
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
max_budget: 100 # Optional[float]: $100 budget for the team
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
- "/team/daily/activity" # Allow members to view team usage
```
### 3.2 Auto-create a new team on LiteLLM

2
docs/my-website/docusaurus.config.js
View file

@ -284,8 +284,8 @@ const config = {
label: 'Enterprise',
to: "docs/enterprise"
},
{ to: '/release_notes', label: 'Changelog', position: 'left' },
{ to: '/blog', label: 'Blog', position: 'left' },
{ to: '/release_notes', label: 'Release Notes', position: 'left' },
{
href: 'https://github.com/BerriAI/litellm',
position: 'right',

BIN
docs/my-website/img/april_townhall_banner.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 49 KiB

BIN
docs/my-website/img/release_notes/mcp_toolsets.jpeg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 111 KiB

BIN
docs/my-website/img/release_notes/skills_marketplace.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1 MiB

25
docs/my-website/package-lock.json generated
View file

@ -13,18 +13,18 @@
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
"prism-react-renderer": "^1.3.5",
"react": "^18.0.0 || ^19.0.0",
"react-dom": "^18.0.0 || ^19.0.0",
"sharp": "^0.32.6",
"uuid": "^9.0.1"
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",
"react-dom": "18.3.1",
"sharp": "0.32.6",
"uuid": "9.0.1"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "3.8.1",
"dotenv": "^16.4.5"
"dotenv": "16.6.1"
},
"engines": {
"node": ">=16.14",
@ -20403,13 +20403,6 @@
"url": "https://opencollective.com/webpack"
}
},
"node_modules/search-insights": {
"version": "2.17.3",
"resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz",
"integrity": "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==",
"license": "MIT",
"peer": true
},
"node_modules/section-matter": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz",

18
docs/my-website/package.json
View file

@ -19,18 +19,18 @@
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
"prism-react-renderer": "^1.3.5",
"react": "^18.0.0 || ^19.0.0",
"react-dom": "^18.0.0 || ^19.0.0",
"sharp": "^0.32.6",
"uuid": "^9.0.1"
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",
"react-dom": "18.3.1",
"sharp": "0.32.6",
"uuid": "9.0.1"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "3.8.1",
"dotenv": "^16.4.5"
"dotenv": "16.6.1"
},
"browserslist": {
"production": [

236
docs/my-website/release_notes/v1.83.3/index.md Normal file
View file

@ -0,0 +1,236 @@
---
title: "[Preview] v1.83.3.rc.1 - Introducing MCP Skills Marketplace"
slug: "v1-83-3-rc-1"
date: 2026-04-04T00:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Ryan Crabbe
title: Full Stack Engineer, LiteLLM
url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
- name: Yuneng Jiang
title: Senior Full Stack Engineer, LiteLLM
url: https://www.linkedin.com/in/yuneng-david-jiang-455676139/
image_url: https://avatars.githubusercontent.com/u/171294688?v=4
- name: Shivam Rawat
title: Forward Deployed Engineer, LiteLLM
url: https://linkedin.com/in/shivam-rawat-482937318
image_url: https://github.com/shivamrawat1.png
hide_table_of_contents: false
---
## Deploy this version
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
<Tabs>
<TabItem value="docker" label="Docker">
```bash
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:main-v1.83.3.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
```bash
pip install litellm==1.83.3rc1
```
</TabItem>
</Tabs>
## Key Highlights
- **MCP Toolsets** — [Create curated tool subsets from one or more MCP servers with scoped permissions, and manage them from the UI or API](../../docs/mcp)
- **Skills Marketplace** — [Browse, install, and publish Claude Code skills from a self-hosted marketplace — works across Anthropic, Vertex AI, Azure, and Bedrock](../../docs/proxy/skills)
- **Guardrail Fallbacks** — [Configure `on_error` behavior so guardrail failures degrade gracefully instead of blocking the request](../../docs/proxy/guardrails)
- **Team Bring Your Own Guardrails** — [Teams can now attach and manage their own guardrails directly from team settings in the UI](../../docs/proxy/guardrails)
---
### Skills Marketplace
The Skills Marketplace gives teams a self-hosted catalog for discovering, installing, and publishing Claude Code skills. Skills are portable across Anthropic, Vertex AI, Azure, and Bedrock — so a skill published once works everywhere your gateway routes to.
![Skills Marketplace](../../img/release_notes/skills_marketplace.png)
[Get Started](../../docs/proxy/skills)
### Guardrail Fallbacks
Guardrail pipelines now support an optional `on_error` behavior. When a guardrail check fails or errors out, you can configure the pipeline to fall back gracefully — logging the failure and continuing the request — instead of returning a hard 500 to the caller. This is especially useful for non-critical guardrails where availability matters more than enforcement.
### Team Bring Your Own Guardrails
Teams can now attach guardrails directly from the team management UI. Admins configure available guardrails at the project or proxy level, and individual teams select which ones apply to their traffic — no config file changes or proxy restarts needed. This also ships with project-level guardrail support in the project create/edit flows.
### MCP Toolsets
MCP Toolsets let AI platform admins create curated subsets of tools from one or more MCP servers and assign them to teams and keys with scoped permissions. Instead of granting access to an entire MCP server, you can now bundle specific tools into a named toolset — controlling exactly which tools each team or API key can invoke. Toolsets are fully managed through the UI (new Toolsets tab) and API, and work seamlessly with the Responses API and Playground.
![MCP Toolsets](../../img/release_notes/mcp_toolsets.jpeg)
[Get Started](../../docs/mcp)
---
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| Brave Search | `brave/search` | - | - | - | Search tool integration metadata in cost map ([PR #25042](https://github.com/BerriAI/litellm/pull/25042)) |
| AWS Bedrock | `nvidia.nemotron-super-3-120b` | 256K | Added | Added | Chat completions, function calling, system messages ([PR #24588](https://github.com/BerriAI/litellm/pull/24588)) |
| OCI GenAI | Multiple new chat + embedding entries | Varies | Updated | Updated | Expanded chat + embedding model catalog |
#### Features
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Add Nova Canvas image edit support - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24869](https://github.com/BerriAI/litellm/pull/24869)
- Improve cache usage exposure for Claude-compatible streaming paths - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24850](https://github.com/BerriAI/litellm/pull/24850)
- Bedrock model catalog updates - [PR #24645](https://github.com/BerriAI/litellm/pull/24645)
- **[OCI GenAI](../../docs/providers/oci)**
- Add native embeddings support + expanded model catalog - [PR #25151](https://github.com/BerriAI/litellm/pull/25151), [PR #24887](https://github.com/BerriAI/litellm/pull/24887)
- **[Google Vertex AI](../../docs/providers/vertex)**
- Add unversioned Claude Haiku pricing entry to ensure accurate spend accounting - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
### Bug Fixes
- **General**
- Fix `gpt-5.4` pricing metadata - [PR #24748](https://github.com/BerriAI/litellm/pull/24748)
- Fix gov pricing tests and Bedrock model test follow-ups - [PR #25022](https://github.com/BerriAI/litellm/pull/25022), [PR #24947](https://github.com/BerriAI/litellm/pull/24947), [PR #24931](https://github.com/BerriAI/litellm/pull/24931)
## LLM API Endpoints
#### Features
- **[A2A / MCP Gateway API (/a2a, /mcp)](../../docs/mcp)**
- Preserve JSON-RPC envelope for AgentCore A2A-native agents - [PR #25092](https://github.com/BerriAI/litellm/pull/25092)
- Bedrock Anthropic file/document handling fix from internal staging - [PR #25050](https://github.com/BerriAI/litellm/pull/25050), [PR #25047](https://github.com/BerriAI/litellm/pull/25047)
#### Bugs
- **[Search API (/search)](../../docs/search)**
- Support self-hosted Firecrawl response format in search transforms - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24866](https://github.com/BerriAI/litellm/pull/24866)
## Management Endpoints / UI
#### Features
- **Virtual Keys**
- Add substring search for `user_id` and `key_alias` on `/key/list` - [PR #24751](https://github.com/BerriAI/litellm/pull/24751), [PR #24746](https://github.com/BerriAI/litellm/pull/24746)
- Wire `team_id` filter to key alias dropdown on Virtual Keys tab - [PR #25119](https://github.com/BerriAI/litellm/pull/25119), [PR #25114](https://github.com/BerriAI/litellm/pull/25114)
- Allow hashed `token_id` in `/key/update` endpoint - [PR #24969](https://github.com/BerriAI/litellm/pull/24969)
- **Teams + Organizations**
- Resolve access-group models/MCP servers/agents in team endpoints and UI - [PR #25119](https://github.com/BerriAI/litellm/pull/25119), [PR #25027](https://github.com/BerriAI/litellm/pull/25027)
- Allow changing team organization from team settings - [PR #25095](https://github.com/BerriAI/litellm/pull/25095)
- Add per-model rate limits to team edit/info views - [PR #25156](https://github.com/BerriAI/litellm/pull/25156), [PR #25144](https://github.com/BerriAI/litellm/pull/25144)
- **Usage + Analytics**
- Add paginated team search on usage page filters - [PR #25107](https://github.com/BerriAI/litellm/pull/25107)
- Use entity key for usage export display correctness - [PR #25153](https://github.com/BerriAI/litellm/pull/25153)
- **Models + Providers**
- Include access-group models in UI model listing - [PR #24743](https://github.com/BerriAI/litellm/pull/24743)
- Expose Azure Entra ID credential fields in provider forms - [PR #25137](https://github.com/BerriAI/litellm/pull/25137)
- Do not inject `vector_store_ids: []` when editing a model - [PR #25133](https://github.com/BerriAI/litellm/pull/25133)
- **Guardrails UI**
- Add project-level guardrails support in project create/edit flows - [PR #25100](https://github.com/BerriAI/litellm/pull/25100)
- Allow adding team guardrails from the UI - [PR #25038](https://github.com/BerriAI/litellm/pull/25038)
- **UI Cleanup**
- Migrate Tremor Text/Badge to antd Tag and native spans - [PR #24750](https://github.com/BerriAI/litellm/pull/24750)
#### Bugs
- Fix logs page showing unfiltered results when backend filter returns zero rows - [PR #24745](https://github.com/BerriAI/litellm/pull/24745)
- Enforce upperbound key params on `/key/update` and bulk update hook paths - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #25103](https://github.com/BerriAI/litellm/pull/25103)
- Fix team model update 500 due to unsupported Prisma JSON path filter - [PR #25152](https://github.com/BerriAI/litellm/pull/25152)
## AI Integrations
### Logging
- **General**
- Eliminate race condition in streaming `guardrail_information` logging - [PR #24592](https://github.com/BerriAI/litellm/pull/24592)
- Use actual `start_time` in failed request spend logs - [PR #24906](https://github.com/BerriAI/litellm/pull/24906)
- Harden credential redaction + stop logging raw sensitive auth values - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
### Guardrails
- Add optional `on_error` for guardrail pipeline failures - [PR #25150](https://github.com/BerriAI/litellm/pull/25150), [PR #24831](https://github.com/BerriAI/litellm/pull/24831)
- Return HTTP 400 (vs 500) for Model Armor streaming blocks - [PR #24693](https://github.com/BerriAI/litellm/pull/24693)
### Prompt Management
- Add environment + user tracking for prompts (`development/staging/production`) in CRUD + UI flows - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24855](https://github.com/BerriAI/litellm/pull/24855)
### Secret Managers
- No major new secret manager provider additions in this RC.
## Spend Tracking, Budgets and Rate Limiting
- Enforce budget for models not directly present in the cost map - [PR #24949](https://github.com/BerriAI/litellm/pull/24949)
- Add per-model rate limits in team settings/info UI - [PR #25144](https://github.com/BerriAI/litellm/pull/25144)
- Fix unversioned Vertex Claude Haiku pricing entry to avoid `$0.00` accounting - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
## MCP Gateway
- Introduce **MCP Toolsets** with DB types, CRUD APIs, scoped permissions, and UI management tab - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
- Resolve toolset names and enforce toolset access correctly in Responses API and streamable MCP paths - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
- Switch toolset permission caching to shared cache path and improve cache invalidation behavior - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
- Allow JWT auth for `/v1/mcp/server/*` sub-paths - [PR #25113](https://github.com/BerriAI/litellm/pull/25113), [PR #24698](https://github.com/BerriAI/litellm/pull/24698)
- Add STS AssumeRole support for MCP SigV4 auth - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
- Add tag query fix + MCP metadata support cherry-pick - [PR #25145](https://github.com/BerriAI/litellm/pull/25145)
## Performance / Loadbalancing / Reliability improvements
- Integrate router health-check failures with cooldown behavior and transient 429/408 handling - [PR #25150](https://github.com/BerriAI/litellm/pull/25150), [PR #24988](https://github.com/BerriAI/litellm/pull/24988)
- Add distributed lock for key rotation job execution - [PR #25150](https://github.com/BerriAI/litellm/pull/25150), [PR #23364](https://github.com/BerriAI/litellm/pull/23364), [PR #23834](https://github.com/BerriAI/litellm/pull/23834)
- Improve team routing reliability with deterministic grouping, isolation fixes, stale alias controls, and order-based fallback - [PR #25154](https://github.com/BerriAI/litellm/pull/25154), [PR #25148](https://github.com/BerriAI/litellm/pull/25148)
- Regenerate GCP IAM token per async Redis cluster connection (fix token TTL failures) - [PR #25155](https://github.com/BerriAI/litellm/pull/25155), [PR #24426](https://github.com/BerriAI/litellm/pull/24426)
- Restore MCP server fields dropped by schema sync migration - [PR #24078](https://github.com/BerriAI/litellm/pull/24078)
- Proxy server reliability hardening with bounded queue usage - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
## Documentation Updates
- Improve HA control plane diagram clarity + mobile rendering updates - [PR #24747](https://github.com/BerriAI/litellm/pull/24747)
- Document `default_team_params` in config reference and examples - [PR #25032](https://github.com/BerriAI/litellm/pull/25032)
- Add JWT to Virtual Key mapping guide - [PR #24882](https://github.com/BerriAI/litellm/pull/24882)
- Add MCP Toolsets docs and sidebar updates - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
- Security docs updates and April hardening blog - [PR #24867](https://github.com/BerriAI/litellm/pull/24867), [PR #24868](https://github.com/BerriAI/litellm/pull/24868), [PR #24871](https://github.com/BerriAI/litellm/pull/24871), [PR #25102](https://github.com/BerriAI/litellm/pull/25102)
- General docs cleanup + townhall announcement updates - [PR #24839](https://github.com/BerriAI/litellm/pull/24839), [PR #25026](https://github.com/BerriAI/litellm/pull/25026), [PR #25021](https://github.com/BerriAI/litellm/pull/25021)
## Infrastructure / Security Notes
- Harden npm and Docker supply chain workflows and release pipeline checks - [PR #24838](https://github.com/BerriAI/litellm/pull/24838), [PR #24877](https://github.com/BerriAI/litellm/pull/24877), [PR #24881](https://github.com/BerriAI/litellm/pull/24881), [PR #24905](https://github.com/BerriAI/litellm/pull/24905), [PR #24951](https://github.com/BerriAI/litellm/pull/24951), [PR #25023](https://github.com/BerriAI/litellm/pull/25023), [PR #25034](https://github.com/BerriAI/litellm/pull/25034), [PR #25036](https://github.com/BerriAI/litellm/pull/25036), [PR #25037](https://github.com/BerriAI/litellm/pull/25037), [PR #25136](https://github.com/BerriAI/litellm/pull/25136), [PR #25158](https://github.com/BerriAI/litellm/pull/25158)
- Resolve CodeQL/security workflow issues and fix broken action SHA references - [PR #24880](https://github.com/BerriAI/litellm/pull/24880), [PR #24815](https://github.com/BerriAI/litellm/pull/24815)
- Re-add Codecov reporting in GHA matrix workflows - [PR #24804](https://github.com/BerriAI/litellm/pull/24804)
- Fix(docker): load enterprise hooks in non-root runtime image - [PR #24917](https://github.com/BerriAI/litellm/pull/24917)
- Apply Black formatting to 14 files - [PR #24532](https://github.com/BerriAI/litellm/pull/24532)
- Fix lint issues - [PR #24932](https://github.com/BerriAI/litellm/pull/24932)
## New Contributors
* @vanhtuan0409 made their first contribution in https://github.com/BerriAI/litellm/pull/24078
* @clfhhc made their first contribution in https://github.com/BerriAI/litellm/pull/24932
**Full Changelog**: https://github.com/BerriAI/litellm/compare/v1.83.0-nightly...v1.83.3.rc.1

8
docs/my-website/sidebars.js
View file

@ -325,6 +325,7 @@ const sidebars = {
"mcp_control",
"mcp_cost",
"mcp_guardrail",
"mcp_toolsets",
{
type: "link",
label: "MCP Troubleshooting Guide",
@ -348,6 +349,7 @@ const sidebars = {
"proxy/debugging",
"proxy/error_diagnosis",
"proxy/deploy",
"proxy/docker_image_security",
"proxy/health",
"proxy/master_key_rotations",
"proxy/model_management",
@ -562,7 +564,8 @@ const sidebars = {
"proxy/model_access",
"proxy/model_access_groups",
"proxy/access_groups",
"proxy/team_model_add"
"proxy/team_model_add",
"proxy/credential_routing"
]
},
{
@ -1051,7 +1054,8 @@ const sidebars = {
"proxy/fallback_management",
"proxy/tag_routing",
"proxy/timeout",
"wildcard_routing"
"wildcard_routing",
"proxy/health_check_routing"
],
},
{

2
enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py
View file

@ -114,6 +114,7 @@ class PagerDutyAlerting(SlackAlerting):
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
user_api_key_org_id=_meta.get("user_api_key_org_id"),
user_api_key_org_alias=_meta.get("user_api_key_org_alias"),
user_api_key_team_id=_meta.get("user_api_key_team_id"),
user_api_key_project_id=_meta.get("user_api_key_project_id"),
user_api_key_project_alias=_meta.get("user_api_key_project_alias"),
@ -196,6 +197,7 @@ class PagerDutyAlerting(SlackAlerting):
else None
),
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_org_alias=user_api_key_dict.organization_alias,
user_api_key_team_id=user_api_key_dict.team_id,
user_api_key_project_id=user_api_key_dict.project_id,
user_api_key_project_alias=user_api_key_dict.project_alias,

45
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py
View file

@ -11,6 +11,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.constants import (
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
MAX_OBJECTS_PER_POLL_CYCLE,
STALE_OBJECT_CLEANUP_BATCH_SIZE,
)
if TYPE_CHECKING:
@ -32,21 +33,49 @@ class CheckResponsesCost:
self.prisma_client: PrismaClient = prisma_client
self.llm_router: Router = llm_router
async def _expire_stale_rows(
self, cutoff: datetime, batch_size: int
) -> int:
"""Execute the bounded UPDATE that marks stale rows as 'stale_expired'.
Isolated so it can be swapped / mocked in tests without touching the
orchestration logic in ``_cleanup_stale_managed_objects``.
Uses PostgreSQL syntax (``$1::timestamptz``, ``LIMIT``, double-quoted
identifiers) which is the only dialect the proxy supports — every
``schema.prisma`` in the repo sets ``provider = "postgresql"``.
Same pattern as ``spend_log_cleanup.py``.
"""
return await self.prisma_client.db.execute_raw(
"""
UPDATE "LiteLLM_ManagedObjectTable"
SET "status" = 'stale_expired'
WHERE "id" IN (
SELECT "id" FROM "LiteLLM_ManagedObjectTable"
WHERE "file_purpose" = 'response'
AND "status" NOT IN ('completed', 'complete', 'failed', 'expired', 'cancelled', 'stale_expired')
AND "created_at" < $1::timestamptz
ORDER BY "created_at" ASC
LIMIT $2
)
""",
cutoff,
batch_size,
)
async def _cleanup_stale_managed_objects(self) -> None:
"""
Mark managed objects older than MANAGED_OBJECT_STALENESS_CUTOFF_DAYS days
in non-terminal states as 'stale_expired'. These will never complete and
should not be polled.
Runs as a single DB query with a subquery LIMIT so no rows are loaded
into Python memory. Processes at most STALE_OBJECT_CLEANUP_BATCH_SIZE
rows per invocation to avoid overwhelming the DB when there is a large
backlog.
"""
cutoff = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
result = await self.prisma_client.db.litellm_managedobjecttable.update_many(
where={
"file_purpose": "response",
"status": {"not_in": ["completed", "complete", "failed", "expired", "cancelled", "stale_expired"]},
"created_at": {"lt": cutoff},
},
data={"status": "stale_expired"},
)
result = await self._expire_stale_rows(cutoff, STALE_OBJECT_CLEANUP_BATCH_SIZE)
if result > 0:
verbose_proxy_logger.warning(
f"CheckResponsesCost: marked {result} stale managed objects "

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.35"
version = "0.1.36"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.35"
version = "0.1.36"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",

8
litellm-js/proxy/package.json
View file

@ -4,11 +4,11 @@
"deploy": "wrangler deploy --minify src/index.ts"
},
"dependencies": {
"hono": "^4.1.4",
"openai": "^4.29.2"
"hono": "4.1.4",
"openai": "4.29.2"
},
"devDependencies": {
"@cloudflare/workers-types": "^4.20240208.0",
"wrangler": "^3.32.0"
"@cloudflare/workers-types": "4.20240208.0",
"wrangler": "3.32.0"
}
}

8
litellm-js/spend-logs/package-lock.json generated
View file

@ -5,12 +5,12 @@
"packages": {
"": {
"dependencies": {
"@hono/node-server": "^1.10.1",
"hono": "^4.12.7"
"@hono/node-server": "1.19.6",
"hono": "4.12.7"
},
"devDependencies": {
"@types/node": "^20.11.17",
"tsx": "^4.7.1"
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
},
"node_modules/@esbuild/aix-ppc64": {

8
litellm-js/spend-logs/package.json
View file

@ -3,11 +3,11 @@
"dev": "tsx watch src/index.ts"
},
"dependencies": {
"@hono/node-server": "^1.10.1",
"hono": "^4.12.7"
"@hono/node-server": "1.19.6",
"hono": "4.12.7"
},
"devDependencies": {
"@types/node": "^20.11.17",
"tsx": "^4.7.1"
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
}

19
litellm-proxy-extras/litellm_proxy_extras/migrations/20260321000000_add_mcp_toolsets/migration.sql Normal file
View file

@ -0,0 +1,19 @@
-- CreateTable: LiteLLM_MCPToolsetTable
CREATE TABLE IF NOT EXISTS "LiteLLM_MCPToolsetTable" (
"toolset_id" TEXT NOT NULL,
"toolset_name" TEXT NOT NULL,
"description" TEXT,
"tools" JSONB NOT NULL DEFAULT '[]',
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT,
CONSTRAINT "LiteLLM_MCPToolsetTable_pkey" PRIMARY KEY ("toolset_id")
);
-- CreateIndex
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_MCPToolsetTable_toolset_name_key" ON "LiteLLM_MCPToolsetTable"("toolset_name");
-- AlterTable: add mcp_toolsets to ObjectPermissionTable
ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "mcp_toolsets" TEXT[] DEFAULT ARRAY[]::TEXT[];

12
litellm-proxy-extras/litellm_proxy_extras/migrations/20260331000000_add_prompt_environment_and_created_by/migration.sql Normal file
View file

@ -0,0 +1,12 @@
-- AlterTable
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "environment" TEXT NOT NULL DEFAULT 'development';
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "created_by" TEXT;
-- DropIndex (old unique constraint)
DROP INDEX IF EXISTS "LiteLLM_PromptTable_prompt_id_version_key";
-- CreateIndex (new unique constraint)
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_environment_key" ON "LiteLLM_PromptTable"("prompt_id", "version", "environment");
-- CreateIndex (new composite index)
CREATE INDEX "LiteLLM_PromptTable_prompt_id_environment_idx" ON "LiteLLM_PromptTable"("prompt_id", "environment");

29
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -273,6 +273,7 @@ model LiteLLM_ObjectPermissionTable {
agent_access_groups String[] @default([])
models String[] @default([])
blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission
mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user
teams LiteLLM_TeamTable[]
projects LiteLLM_ProjectTable[]
verification_tokens LiteLLM_VerificationToken[]
@ -321,15 +322,28 @@ model LiteLLM_MCPServerTable {
byok_description String[] @default([])
byok_api_key_help_url String?
source_url String?
approval_status String? @default("active")
submitted_by String?
submitted_at DateTime?
reviewed_at DateTime?
review_notes String?
// BYOM submission lifecycle
approval_status String? @default("active")
submitted_by String?
submitted_at DateTime?
reviewed_at DateTime?
review_notes String?
@@index([approval_status])
}
// Named collection of {server_id, tool_name} pairs that can be granted to keys/teams
model LiteLLM_MCPToolsetTable {
toolset_id String @id @default(uuid())
toolset_name String @unique
description String?
tools Json @default("[]") // [{server_id: string, tool_name: string}]
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}
// Per-user BYOK credentials for MCP servers
model LiteLLM_MCPUserCredentials {
id String @id @default(uuid())
@ -1001,12 +1015,15 @@ model LiteLLM_PromptTable {
id String @id @default(uuid())
prompt_id String
version Int @default(1)
environment String @default("development")
created_by String?
litellm_params Json
prompt_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([prompt_id, version])
@@unique([prompt_id, version, environment])
@@index([prompt_id, environment])
@@index([prompt_id])
}

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.62"
version = "0.4.65"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.62"
version = "0.4.65"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

2
litellm/__init__.py
View file

@ -318,6 +318,7 @@ return_response_headers: bool = (
False # get response headers from LLM Api providers - example x-remaining-requests,
)
enable_json_schema_validation: bool = False
enable_model_config_credential_overrides: bool = False
enable_key_alias_format_validation: bool = (
False # opt-in validation of key_alias format on /key/generate and /key/update
)
@ -1838,6 +1839,7 @@ if TYPE_CHECKING:
)
from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig
from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig
from .llms.oci.embed.transformation import OCIEmbeddingConfig as OCIEmbeddingConfig
from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig
from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig
from .llms.lambda_ai.chat.transformation import (

44
litellm/_logging.py
View file

@ -26,6 +26,12 @@ _REDACTED = "REDACTED"
def _build_secret_patterns() -> re.Pattern:
patterns: List[str] = [
# ── PEM private key / certificate blocks ──
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
# ── GCP OAuth2 access tokens (ya29.*) ──
r"\bya29\.[A-Za-z0-9_.~+/-]+",
# ── Credential %s formatting (space separator, no key= prefix) ──
r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
# AWS access key IDs
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
# AWS secrets / session tokens / access key IDs (key=value)
@ -46,7 +52,8 @@ def _build_secret_patterns() -> re.Pattern:
# Google API keys
r"AIza[0-9A-Za-z\-_]{35}",
# Password / secret params (handles key=value and 'key': 'value')
r"\w*(?:password|passwd|client_secret|secret_key|_secret)"
# Word boundary prevents O(n^2) backtracking on long word-char runs.
r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
# Database connection string credentials (scheme://user:pass@host)
r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
@ -56,13 +63,21 @@ def _build_secret_patterns() -> re.Pattern:
# Catches secrets inside dicts/config dumps by matching on the KEY name
# regardless of what the value looks like.
# e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
# private_key with PEM-aware value capture
r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
r"(?:master_key|database_url|db_url|connection_string|"
r"private_key|signing_key|encryption_key|"
r"signing_key|encryption_key|"
r"auth_token|access_token|refresh_token|"
r"slack_webhook_url|webhook_url|"
r"database_connection_string|"
r"huggingface_token|jwt_secret)"
r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
# ── Raw JWTs (without Bearer prefix) ──
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
# ── Azure SAS tokens in URLs ──
r"[?&]sig=[A-Za-z0-9%+/=]+",
# ── Full JSON service-account blobs (single-line and multi-line) ──
r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
]
return re.compile("|".join(patterns), re.IGNORECASE)
@ -74,6 +89,23 @@ def _redact_string(value: str) -> str:
return _SECRET_RE.sub(_REDACTED, value)
def redact_secrets(value: str) -> str:
"""Public API: redact known secret/credential patterns from an arbitrary string.
Use this for code paths that bypass the logging system — e.g. Slack/Teams
alerting, HTTP error response bodies, or any other string that may contain
secrets and will be sent to an external sink.
Not to be confused with redact_message_input_output_from_logging() in
litellm_core_utils/redact_messages.py, which redacts LLM prompt/response
content for privacy — this function redacts credential patterns (API keys,
PEM blocks, tokens, etc.) by shape.
"""
if not _ENABLE_SECRET_REDACTION:
return value
return _redact_string(value)
class SecretRedactionFilter(logging.Filter):
"""Scrubs known secret/credential patterns from log records."""
@ -211,6 +243,12 @@ class JsonFormatter(Formatter):
if key not in _STANDARD_RECORD_ATTRS and key not in json_record:
json_record[key] = value
# Set component/logger only if not already supplied via extra={...}
if "component" not in json_record:
json_record["component"] = record.name
if "logger" not in json_record:
json_record["logger"] = f"{record.filename}:{record.lineno}"
if record.exc_info:
json_record["stacktrace"] = record.exc_text or self.formatException(
record.exc_info
@ -441,7 +479,7 @@ def _enable_debugging():
def print_verbose(print_statement):
try:
if set_verbose:
print(print_statement) # noqa
print(redact_secrets(str(print_statement))) # noqa
except Exception:
pass

71
litellm/_redis.py
View file

@ -18,6 +18,10 @@ import redis # type: ignore
import redis.asyncio as async_redis # type: ignore
from litellm import get_secret, get_secret_str
from litellm._redis_credential_provider import (
GCPIAMCredentialProvider,
_generate_gcp_iam_access_token,
)
from litellm.constants import REDIS_CONNECTION_POOL_TIMEOUT, REDIS_SOCKET_TIMEOUT
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
@ -107,33 +111,6 @@ def _redis_kwargs_from_environment():
return return_dict
def _generate_gcp_iam_access_token(service_account: str) -> str:
"""
Generate GCP IAM access token for Redis authentication.
Args:
service_account: GCP service account in format 'projects/-/serviceAccounts/name@project.iam.gserviceaccount.com'
Returns:
Access token string for GCP IAM authentication
"""
try:
from google.cloud import iam_credentials_v1
except ImportError:
raise ImportError(
"google-cloud-iam is required for GCP IAM Redis authentication. "
"Install it with: pip install google-cloud-iam"
)
client = iam_credentials_v1.IAMCredentialsClient()
request = iam_credentials_v1.GenerateAccessTokenRequest(
name=service_account,
scope=["https://www.googleapis.com/auth/cloud-platform"],
)
response = client.generate_access_token(request=request)
return str(response.access_token)
def create_gcp_iam_redis_connect_func(
service_account: str,
ssl_ca_certs: Optional[str] = None,
@ -266,7 +243,7 @@ def _get_redis_client_logic(**env_overrides):
service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
)
# Store GCP service account in redis_connect_func for async cluster access
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account # type: ignore[attr-defined]
# Remove GCP-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("gcp_service_account", None)
@ -413,41 +390,13 @@ def get_redis_async_client(
# Handle GCP IAM authentication for async clusters
redis_connect_func = cluster_kwargs.pop("redis_connect_func", None)
from litellm import get_secret_str
# Get GCP service account - first try from redis_connect_func, then from environment
gcp_service_account = None
# Use a CredentialProvider so the IAM token is regenerated on every new
# connection — mirrors the sync path where redis_connect_func is invoked
# per connection. Without this, the token would expire after ~1 hour.
if redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
gcp_service_account = redis_connect_func._gcp_service_account
else:
gcp_service_account = redis_kwargs.get(
"gcp_service_account"
) or get_secret_str("REDIS_GCP_SERVICE_ACCOUNT")
verbose_logger.debug(
f"DEBUG: Redis cluster kwargs: redis_connect_func={redis_connect_func is not None}, gcp_service_account_provided={gcp_service_account is not None}"
)
# If GCP IAM is configured (indicated by redis_connect_func), generate access token and use as password
if redis_connect_func and gcp_service_account:
verbose_logger.debug(
"DEBUG: Generating IAM token for service account (value not logged for security reasons)"
)
try:
# Generate IAM access token using the helper function
access_token = _generate_gcp_iam_access_token(gcp_service_account)
cluster_kwargs["password"] = access_token
verbose_logger.debug(
"DEBUG: Successfully generated GCP IAM access token for async Redis cluster"
)
except Exception as e:
verbose_logger.error(f"Failed to generate GCP IAM access token: {e}")
from redis.exceptions import AuthenticationError
raise AuthenticationError("Failed to generate GCP IAM access token")
else:
verbose_logger.debug(
f"DEBUG: Not using GCP IAM auth - redis_connect_func={redis_connect_func is not None}, gcp_service_account_provided={gcp_service_account is not None}"
cluster_kwargs["credential_provider"] = GCPIAMCredentialProvider(
redis_connect_func._gcp_service_account
)
new_startup_nodes: List[ClusterNode] = []

53
litellm/_redis_credential_provider.py Normal file
View file

@ -0,0 +1,53 @@
import asyncio
from typing import Tuple
from redis.credentials import CredentialProvider # type: ignore[attr-defined]
def _generate_gcp_iam_access_token(service_account: str) -> str:
"""
Generate GCP IAM access token for Redis authentication.
Args:
service_account: GCP service account in format 'projects/-/serviceAccounts/name@project.iam.gserviceaccount.com'
Returns:
Access token string for GCP IAM authentication
"""
try:
from google.cloud import iam_credentials_v1
except ImportError:
raise ImportError(
"google-cloud-iam is required for GCP IAM Redis authentication. "
"Install it with: pip install google-cloud-iam"
)
client = iam_credentials_v1.IAMCredentialsClient()
request = iam_credentials_v1.GenerateAccessTokenRequest(
name=service_account,
scope=["https://www.googleapis.com/auth/cloud-platform"],
)
response = client.generate_access_token(request=request)
return str(response.access_token)
class GCPIAMCredentialProvider(CredentialProvider):
"""
redis.credentials.CredentialProvider implementation that generates a fresh GCP IAM
token on every new connection. This fixes the 1-hour token expiry issue for async
Redis cluster clients, which previously generated the token once at startup and
cached it as a static password.
"""
def __init__(self, gcp_service_account: str) -> None:
self._gcp_service_account = gcp_service_account
def get_credentials(self) -> Tuple[str]:
token = _generate_gcp_iam_access_token(self._gcp_service_account)
return (token,)
async def get_credentials_async(self) -> Tuple[str]:
token = await asyncio.to_thread(
_generate_gcp_iam_access_token, self._gcp_service_account
)
return (token,)

14
litellm/a2a_protocol/litellm_completion_bridge/handler.py
View file

@ -48,20 +48,19 @@ class A2ACompletionBridgeHandler:
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
# If provider config exists, use it
if a2a_provider_config is not None:
if api_base is None:
raise ValueError(f"api_base is required for {custom_llm_provider}")
verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider}")
response_data = await a2a_provider_config.handle_non_streaming(
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
)
return response_data
@ -147,14 +146,12 @@ class A2ACompletionBridgeHandler:
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
# If provider config exists, use it
if a2a_provider_config is not None:
if api_base is None:
raise ValueError(f"api_base is required for {custom_llm_provider}")
verbose_logger.info(
f"A2A: Using provider config for {custom_llm_provider} (streaming)"
)
@ -163,6 +160,7 @@ class A2ACompletionBridgeHandler:
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
):
yield chunk

6
litellm/a2a_protocol/providers/base.py
View file

@ -3,7 +3,7 @@ Base configuration for A2A protocol providers.
"""
from abc import ABC, abstractmethod
from typing import Any, AsyncIterator, Dict
from typing import Any, AsyncIterator, Dict, Optional
class BaseA2AProviderConfig(ABC):
@ -19,7 +19,7 @@ class BaseA2AProviderConfig(ABC):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
"""
@ -41,7 +41,7 @@ class BaseA2AProviderConfig(ABC):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""

22
litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py Normal file
View file

@ -0,0 +1,22 @@
"""
Bedrock AgentCore A2A provider.
Preserves JSON-RPC envelopes for AgentCore agents that speak A2A natively,
bypassing the completion bridge that would otherwise strip the envelope.
"""
from litellm.a2a_protocol.providers.bedrock_agentcore.config import (
BedrockAgentCoreA2AConfig,
)
from litellm.a2a_protocol.providers.bedrock_agentcore.handler import (
BedrockAgentCoreA2AHandler,
)
from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import (
BedrockAgentCoreA2ATransformation,
)
__all__ = [
"BedrockAgentCoreA2AConfig",
"BedrockAgentCoreA2AHandler",
"BedrockAgentCoreA2ATransformation",
]

61
litellm/a2a_protocol/providers/bedrock_agentcore/config.py Normal file
View file

@ -0,0 +1,61 @@
"""
Bedrock AgentCore A2A provider configuration.
"""
from typing import Any, AsyncIterator, Dict, Optional
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
from litellm.a2a_protocol.providers.bedrock_agentcore.handler import (
BedrockAgentCoreA2AHandler,
)
class BedrockAgentCoreA2AConfig(BaseA2AProviderConfig):
"""
Provider configuration for Bedrock AgentCore A2A-native agents.
AgentCore agents that speak A2A natively expect the full JSON-RPC envelope.
This config bypasses the completion bridge and forwards requests directly,
deriving the endpoint URL from the model ARN and signing with SigV4/JWT.
"""
async def handle_non_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
"""Handle non-streaming request to AgentCore A2A agent."""
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for BedrockAgentCoreA2AConfig "
"(must contain model with AgentCore ARN)"
)
return await BedrockAgentCoreA2AHandler.handle_non_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
)
async def handle_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""Handle streaming request to AgentCore A2A agent."""
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for BedrockAgentCoreA2AConfig "
"(must contain model with AgentCore ARN)"
)
async for chunk in BedrockAgentCoreA2AHandler.handle_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
):
yield chunk

134
litellm/a2a_protocol/providers/bedrock_agentcore/handler.py Normal file
View file

@ -0,0 +1,134 @@
"""
Handler for Bedrock AgentCore A2A-native agents.
Sends JSON-RPC envelopes directly to AgentCore endpoints, bypassing the
completion bridge that would otherwise strip the envelope.
"""
import json
from typing import Any, AsyncIterator, Dict, cast
from litellm._logging import verbose_logger
from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import (
BedrockAgentCoreA2ATransformation,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.custom_http import httpxSpecialProvider
class BedrockAgentCoreA2AHandler:
"""
Handler for Bedrock AgentCore A2A requests.
Constructs JSON-RPC envelopes, signs them via AmazonAgentCoreConfig,
and POSTs directly to the AgentCore endpoint.
"""
@staticmethod
async def handle_non_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
) -> Dict[str, Any]:
"""
Handle non-streaming A2A request to AgentCore.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (model, api_key, etc.)
Returns:
A2A JSON-RPC response dict from the AgentCore agent
"""
url, headers, body = (
BedrockAgentCoreA2ATransformation.get_url_and_signed_request(
request_id=request_id,
params=params,
litellm_params=litellm_params,
method="message/send",
)
)
verbose_logger.info(
f"BedrockAgentCore A2A: Sending non-streaming request to {url}"
)
client = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
)
response = await client.post(
url,
headers=headers,
data=body,
)
response.raise_for_status()
response_data = response.json()
if "error" in response_data:
verbose_logger.warning(
f"BedrockAgentCore A2A: Agent returned error: {response_data['error']}"
)
return response_data
@staticmethod
async def handle_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
) -> AsyncIterator[Dict[str, Any]]:
"""
Handle streaming A2A request to AgentCore.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (model, api_key, etc.)
Yields:
A2A streaming response events from the AgentCore agent
"""
url, headers, body = (
BedrockAgentCoreA2ATransformation.get_url_and_signed_request(
request_id=request_id,
params=params,
litellm_params=litellm_params,
method="message/send",
stream=True,
)
)
verbose_logger.info(
f"BedrockAgentCore A2A: Sending streaming request to {url}"
)
client = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
)
response = await client.post(
url,
headers=headers,
data=body,
stream=True,
)
response.raise_for_status()
# Check content type — AgentCore may return JSON instead of SSE
content_type = response.headers.get("content-type", "").lower()
if "application/json" in content_type:
# Single JSON response fallback (not SSE)
verbose_logger.debug(
"BedrockAgentCore A2A streaming: received JSON instead of SSE, "
"yielding as single event"
)
response_body = await response.aread()
response_data = json.loads(response_body)
yield response_data
else:
# SSE stream — parse data: lines
async for event in BedrockAgentCoreA2ATransformation.parse_sse_events(
response
):
yield event

134
litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py Normal file
View file

@ -0,0 +1,134 @@
"""
Transformation layer for Bedrock AgentCore A2A provider.
Constructs JSON-RPC envelopes, derives AgentCore URLs from model ARNs,
and signs requests via AmazonAgentCoreConfig (SigV4 or JWT).
"""
import json
from typing import Any, AsyncIterator, Dict, Tuple
from litellm._logging import verbose_logger
from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
class BedrockAgentCoreA2ATransformation:
"""
Request/response transformation for Bedrock AgentCore A2A agents.
Reuses AmazonAgentCoreConfig for URL construction, ARN parsing,
and request signing. No logic is duplicated.
"""
@staticmethod
def get_url_and_signed_request(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
method: str = "message/send",
stream: bool = False,
) -> Tuple[str, dict, bytes]:
"""
Build the AgentCore URL, construct a JSON-RPC envelope, and sign the request.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams
litellm_params: Agent's litellm_params (model, api_key, etc.)
method: JSON-RPC method name (default: "message/send")
stream: Whether this is a streaming request
Returns:
Tuple of (url, signed_headers, signed_body_bytes)
"""
# Extract model and strip the "bedrock/" prefix
# "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..."
model = litellm_params.get("model", "")
if model.startswith("bedrock/"):
agentcore_model = model[len("bedrock/") :]
else:
agentcore_model = model
# Build optional_params from litellm_params (everything except model and custom_llm_provider)
optional_params = {
k: v
for k, v in litellm_params.items()
if k not in ("model", "custom_llm_provider")
}
agentcore_config = AmazonAgentCoreConfig()
# Derive URL from ARN
url = agentcore_config.get_complete_url(
api_base=optional_params.get("api_base"),
api_key=optional_params.get("api_key"),
model=agentcore_model,
optional_params=optional_params,
litellm_params=litellm_params,
stream=stream,
)
# Construct JSON-RPC 2.0 envelope
json_rpc_body = {
"jsonrpc": "2.0",
"method": method,
"id": request_id,
"params": params,
}
# Set required AgentCore session headers (normally set by transform_request,
# which we skip because it also builds {"prompt": "..."})
headers: dict = {}
session_id = agentcore_config._get_runtime_session_id(optional_params)
headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = session_id
runtime_user_id = agentcore_config._get_runtime_user_id(optional_params)
if runtime_user_id:
headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] = runtime_user_id
# Sign the request (SigV4 or JWT depending on api_key presence)
signed_headers, signed_body = agentcore_config.sign_request(
headers=headers,
optional_params=optional_params,
request_data=json_rpc_body,
api_base=url,
api_key=optional_params.get("api_key"),
model=agentcore_model,
stream=stream,
)
# sign_request returns Optional[bytes] — ensure we have bytes
if signed_body is None:
signed_body = json.dumps(json_rpc_body).encode()
return url, signed_headers, signed_body
@staticmethod
async def parse_sse_events(response: Any) -> AsyncIterator[Dict[str, Any]]:
"""
Parse SSE events from an httpx streaming response.
Reads line-by-line, parses `data:` lines as JSON, and yields each parsed dict.
Args:
response: httpx streaming response
Yields:
Parsed JSON dicts from SSE data lines
"""
async for line in response.aiter_lines():
line = line.strip()
if not line:
continue
if line.startswith("data:"):
data_str = line[len("data:") :].strip()
if not data_str:
continue
try:
event = json.loads(data_str)
yield event
except json.JSONDecodeError:
verbose_logger.debug(
f"BedrockAgentCore A2A: Skipping non-JSON SSE line: {data_str[:100]}"
)
continue

12
litellm/a2a_protocol/providers/config_manager.py
View file

@ -19,12 +19,14 @@ class A2AProviderConfigManager:
@staticmethod
def get_provider_config(
custom_llm_provider: Optional[str],
model: Optional[str] = None,
) -> Optional[BaseA2AProviderConfig]:
"""
Get the provider configuration for a given custom_llm_provider.
Args:
custom_llm_provider: The provider identifier (e.g., "pydantic_ai_agents")
model: The model string (used to distinguish sub-providers, e.g. agentcore vs other bedrock)
Returns:
Provider configuration instance or None if not found
@ -39,9 +41,11 @@ class A2AProviderConfigManager:
return PydanticAIProviderConfig()
# Add more providers here as needed
# elif custom_llm_provider == "another_provider":
# from litellm.a2a_protocol.providers.another_provider.config import AnotherProviderConfig
# return AnotherProviderConfig()
if custom_llm_provider == "bedrock" and model and "agentcore" in model:
from litellm.a2a_protocol.providers.bedrock_agentcore.config import (
BedrockAgentCoreA2AConfig,
)
return BedrockAgentCoreA2AConfig()
return None

12
litellm/a2a_protocol/providers/pydantic_ai_agents/config.py
View file

@ -2,7 +2,7 @@
Pydantic AI provider configuration.
"""
from typing import Any, AsyncIterator, Dict
from typing import Any, AsyncIterator, Dict, Optional
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
from litellm.a2a_protocol.providers.pydantic_ai_agents.handler import PydanticAIHandler
@ -20,10 +20,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
**kwargs,
api_base: Optional[str] = None,
**kwargs: Any,
) -> Dict[str, Any]:
"""Handle non-streaming request to Pydantic AI agent."""
if api_base is None:
raise ValueError("api_base is required for PydanticAIProviderConfig")
return await PydanticAIHandler.handle_non_streaming(
request_id=request_id,
params=params,
@ -35,10 +37,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""Handle streaming request with fake streaming."""
if not api_base:
raise ValueError("api_base is required for Pydantic AI agents")
async for chunk in PydanticAIHandler.handle_streaming(
request_id=request_id,
params=params,

10
litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py
View file

@ -5,7 +5,7 @@ Pydantic AI agents follow A2A protocol but don't support streaming natively.
This handler provides fake streaming by converting non-streaming responses into streaming chunks.
"""
from typing import Any, AsyncIterator, Dict
from typing import Any, AsyncIterator, Dict, Optional
from litellm._logging import verbose_logger
from litellm.a2a_protocol.providers.pydantic_ai_agents.transformation import (
@ -26,7 +26,7 @@ class PydanticAIHandler:
async def handle_non_streaming(
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
timeout: float = 60.0,
) -> Dict[str, Any]:
"""
@ -41,6 +41,8 @@ class PydanticAIHandler:
Returns:
A2A SendMessageResponse dict
"""
if api_base is None:
raise ValueError("api_base is required for Pydantic AI agents")
verbose_logger.info(f"Pydantic AI: Routing to Pydantic AI agent at {api_base}")
# Send request directly to Pydantic AI agent
@ -57,7 +59,7 @@ class PydanticAIHandler:
async def handle_streaming(
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
timeout: float = 60.0,
chunk_size: int = 50,
delay_ms: int = 10,
@ -80,6 +82,8 @@ class PydanticAIHandler:
Yields:
A2A streaming response events
"""
if api_base is None:
raise ValueError("api_base is required for Pydantic AI agents")
verbose_logger.info(
f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}"
)

32
litellm/constants.py
View file

@ -135,12 +135,32 @@ MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL = int(
MCP_NPM_CACHE_DIR = os.getenv("MCP_NPM_CACHE_DIR", "/tmp/.npm_mcp_cache")
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10"))
# Per-user OAuth token Redis cache (for server-side token storage)
MCP_PER_USER_TOKEN_REDIS_KEY_PREFIX = "mcp:per_user_token"
MCP_PER_USER_TOKEN_DEFAULT_TTL = int(
os.getenv("MCP_PER_USER_TOKEN_DEFAULT_TTL", "43200") # 12 hours
)
MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS = int(
os.getenv("MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS", "60")
)
# MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers.
MCP_CLIENT_TIMEOUT = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0"))
MCP_TOOL_LISTING_TIMEOUT = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0"))
MCP_METADATA_TIMEOUT = float(os.getenv("LITELLM_MCP_METADATA_TIMEOUT", "10.0"))
MCP_HEALTH_CHECK_TIMEOUT = float(os.getenv("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", "10.0"))
# Allowlist of commands permitted for MCP stdio transport.
# Prevents arbitrary command execution via /mcp-rest/test/* endpoints or server creation.
# Note: allowlisted runtimes can still execute code via args (e.g. python -c "...").
# This is an accepted residual risk since these endpoints require PROXY_ADMIN.
# Extend via LITELLM_MCP_STDIO_EXTRA_COMMANDS env var (comma-separated).
_MCP_STDIO_EXTRA_COMMANDS = os.getenv("LITELLM_MCP_STDIO_EXTRA_COMMANDS", "")
MCP_STDIO_ALLOWED_COMMANDS: frozenset = frozenset(
{"npx", "uvx", "python", "python3", "node", "docker", "deno"}
| (set(_MCP_STDIO_EXTRA_COMMANDS.split(",")) - {""})
)
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
"x-litellm-semantic-filter-tools",
@ -1319,6 +1339,9 @@ LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int(
LITELLM_KEY_ROTATION_GRACE_PERIOD: str = os.getenv(
"LITELLM_KEY_ROTATION_GRACE_PERIOD", ""
) # Duration to keep old key valid after rotation (e.g. "24h", "2d"); empty = immediate revoke (default)
LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS = int(
os.getenv("LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS", 600)
) # 10 minutes default — caps the deadlock window if a pod crashes mid-rotation
UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard"
LITELLM_PROXY_ADMIN_NAME = "default_user_id"
@ -1341,12 +1364,14 @@ LITELLM_UI_SESSION_DURATION = os.getenv("LITELLM_UI_SESSION_DURATION", "24h")
########################### DB CRON JOB NAMES ###########################
DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME = "db_daily_tag_spend_update_job"
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics"
CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME = "cloudzero_export_usage_data"
CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(
os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)
)
SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup"
KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job"
SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
@ -1362,6 +1387,9 @@ MAX_OBJECTS_PER_POLL_CYCLE = max(1, int(os.getenv("MAX_OBJECTS_PER_POLL_CYCLE",
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS = max(
1, int(os.getenv("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7))
)
STALE_OBJECT_CLEANUP_BATCH_SIZE = max(
1, int(os.getenv("STALE_OBJECT_CLEANUP_BATCH_SIZE", 1000))
)
# Set PROXY_BATCH_POLLING_ENABLED=false to disable the CheckBatchCost and
# CheckResponsesCost background polling jobs entirely (e.g. to avoid DB load on
# installations with large numbers of stale managed objects).
@ -1393,6 +1421,10 @@ APSCHEDULER_REPLACE_EXISTING = os.getenv(
"1",
] # always replace existing jobs
# The number of tag entries are higher than number of user, team entries. This leads to a higher QPS.
# This will run tag spcific tasks at a later time to smooth QPS
DAILY_TAG_SPEND_BATCH_MULTIPLIER = 2.3
DEFAULT_HEALTH_CHECK_INTERVAL = int(
os.getenv("DEFAULT_HEALTH_CHECK_INTERVAL", 300)
) # 5 minutes

4
litellm/cost_calculator.py
View file

@ -545,8 +545,8 @@ def cost_per_token( # noqa: PLR0915
)
if (
model_info.get("input_cost_per_token", 0) > 0
or model_info.get("output_cost_per_token", 0) > 0
(model_info.get("input_cost_per_token") or 0.0) > 0
or (model_info.get("output_cost_per_token") or 0.0) > 0
):
return generic_cost_per_token(
model=model,

50
litellm/experimental_mcp_client/client.py
View file

@ -82,6 +82,8 @@ class MCPSigV4Auth(httpx.Auth):
aws_session_token: Optional[str] = None,
aws_region_name: Optional[str] = None,
aws_service_name: Optional[str] = None,
aws_role_name: Optional[str] = None,
aws_session_name: Optional[str] = None,
):
try:
from botocore.credentials import Credentials
@ -97,7 +99,16 @@ class MCPSigV4Auth(httpx.Auth):
# Note: os.environ/ prefixed values are already resolved by
# ProxyConfig._check_for_os_environ_vars() at config load time.
# Values arrive here as plain strings.
if aws_access_key_id and aws_secret_access_key:
if aws_role_name:
self.credentials = self._assume_role(
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_region_name=self.region_name,
)
elif aws_access_key_id and aws_secret_access_key:
self.credentials = Credentials(
access_key=aws_access_key_id,
secret_key=aws_secret_access_key,
@ -116,6 +127,43 @@ class MCPSigV4Auth(httpx.Auth):
"(env vars, ~/.aws/credentials, instance profile)."
)
@staticmethod
def _assume_role(
aws_role_name: str,
aws_session_name: Optional[str],
aws_access_key_id: Optional[str],
aws_secret_access_key: Optional[str],
aws_session_token: Optional[str],
aws_region_name: str,
):
"""Call STS AssumeRole and return temporary credentials."""
import boto3
from botocore.credentials import Credentials
session_name = (
aws_session_name or f"litellm-mcp-{int(__import__('time').time())}"
)
sts_kwargs: dict = {"region_name": aws_region_name}
if aws_access_key_id and aws_secret_access_key:
sts_kwargs["aws_access_key_id"] = aws_access_key_id
sts_kwargs["aws_secret_access_key"] = aws_secret_access_key
if aws_session_token:
sts_kwargs["aws_session_token"] = aws_session_token
sts_client = boto3.client("sts", **sts_kwargs)
sts_response = sts_client.assume_role(
RoleArn=aws_role_name,
RoleSessionName=session_name,
)
sts_creds = sts_response["Credentials"]
return Credentials(
access_key=sts_creds["AccessKeyId"],
secret_key=sts_creds["SecretAccessKey"],
token=sts_creds["SessionToken"],
)
def auth_flow(
self, request: httpx.Request
) -> Generator[httpx.Request, httpx.Response, None]:

5
litellm/integrations/azure_storage/azure_storage.py
View file

@ -275,12 +275,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
"""
Gets Azure AD token to use for Azure Storage API requests
"""
verbose_logger.debug("Getting Azure AD Token from Azure Storage")
verbose_logger.debug(
"tenant_id %s, client_id %s, client_secret %s",
"Getting Azure AD Token from Azure Storage, tenant_id=%s, client_id=%s, client_secret=[set=%s]",
tenant_id,
client_id,
client_secret,
client_secret is not None,
)
if tenant_id is None:
raise ValueError(

8
litellm/integrations/gcs_bucket/gcs_bucket_base.py
View file

@ -70,7 +70,9 @@ class GCSBucketBase(CustomBatchLogger):
custom_llm_provider="vertex_ai",
api_base=None,
)
verbose_logger.debug("constructed auth_header %s", auth_header)
verbose_logger.debug(
"constructed auth_header [set=%s]", auth_header is not None
)
headers = {
"Authorization": f"Bearer {auth_header}", # auth_header
"Content-Type": "application/json",
@ -106,7 +108,9 @@ class GCSBucketBase(CustomBatchLogger):
custom_llm_provider="vertex_ai",
api_base=None,
)
verbose_logger.debug("constructed auth_header %s", auth_header)
verbose_logger.debug(
"constructed auth_header [set=%s]", auth_header is not None
)
headers = {
"Authorization": f"Bearer {auth_header}", # auth_header
"Content-Type": "application/json",

9
litellm/integrations/generic_api/generic_api_compatible_callbacks.json
View file

@ -33,5 +33,14 @@
"X-Qualifire-API-Key": "{{environment_variables.QUALIFIRE_API_KEY}}"
},
"environment_variables": ["QUALIFIRE_API_KEY", "QUALIFIRE_WEBHOOK_URL"]
},
"ramp": {
"event_types": ["llm_api_success"],
"endpoint": "https://api.ramp.com/developer/v1/ai-usage/litellm",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.RAMP_API_KEY}}"
},
"environment_variables": ["RAMP_API_KEY"]
}
}

Some files were not shown because too many files have changed in this diff Show more