scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

Merge pull request #20252 from BerriAI/main

Browse source 
merge main
This commit is contained in:
Sameer Kankute 2026-02-02 14:54:50 +05:30 • committed by GitHub
commit f1df5ea9a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1346 changed files with 104266 additions and 12593 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

359
.circleci/config.yml
View file

@ -44,8 +44,8 @@ commands:
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
pip install "pydantic==2.10.2"
pip install "mcp==1.10.1"
pip install "pydantic==2.11.0"
pip install "mcp==1.25.0"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
@ -112,14 +112,14 @@ jobs:
python -m mypy .
cd ..
no_output_timeout: 10m
local_testing:
local_testing_part1:
docker:
- image: cimg/python:3.12
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
parallelism: 4
steps:
- checkout
- setup_google_dns
@ -205,20 +205,32 @@ jobs:
# Run pytest and generate JUnit XML report
- run:
name: Run tests
name: Run tests (Part 1 - A-M)
command: |
pwd
ls
# Add --timeout to kill hanging tests after 300s (5 min)
# Add -v to show test names as they run for debugging
# Add --tb=short for shorter tracebacks
python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=20 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4 --timeout=300 --timeout_method=thread
mkdir test-results
# Discover test files (A-M)
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py")
echo "$TEST_FILES" | circleci tests run \
--split-by=timings \
--verbose \
--command="xargs python -m pytest \
-vv \
--cov=litellm \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
-n 4 \
--timeout=300 \
--timeout_method=thread"
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml local_testing_coverage.xml
mv .coverage local_testing_coverage
mv coverage.xml local_testing_part1_coverage.xml
mv .coverage local_testing_part1_coverage
# Store test results
- store_test_results:
@ -226,8 +238,136 @@ jobs:
- persist_to_workspace:
root: .
paths:
- local_testing_coverage.xml
- local_testing_coverage
- local_testing_part1_coverage.xml
- local_testing_part1_coverage
local_testing_part2:
docker:
- image: cimg/python:3.12
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
parallelism: 4
steps:
- checkout
- setup_google_dns
- run:
name: Show git commit hash
command: |
echo "Git commit hash: $CIRCLE_SHA1"
- restore_cache:
keys:
- v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r .circleci/requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
pip install "langfuse==2.59.7"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.21.1
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "respx==0.22.0"
pip install fastapi
pip install "gunicorn==21.2.0"
pip install "anyio==4.2.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "apscheduler==3.10.4"
pip install "PyGithub==1.59.1"
pip install argon2-cffi
pip install "pytest-mock==3.12.0"
pip install python-multipart
pip install google-cloud-aiplatform
pip install prometheus-client==0.20.0
pip install "pydantic==2.10.2"
pip install "diskcache==5.6.1"
pip install "Pillow==10.3.0"
pip install "jsonschema==4.22.0"
pip install "pytest-xdist==3.6.1"
pip install "pytest-timeout==2.2.0"
pip install "websockets==13.1.0"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
pip uninstall posthog -y
- setup_litellm_enterprise_pip
- save_cache:
paths:
- ./venv
key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- run:
name: Run prisma ./docker/entrypoint.sh
command: |
set +e
chmod +x docker/entrypoint.sh
./docker/entrypoint.sh
set -e
- run:
name: Black Formatting
command: |
cd litellm
python -m pip install black
python -m black .
cd ..
# Run pytest and generate JUnit XML report
- run:
name: Run tests (Part 2 - N-Z)
command: |
mkdir test-results
# Discover test files (N-Z)
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py")
echo "$TEST_FILES" | circleci tests run \
--split-by=timings \
--verbose \
--command="xargs python -m pytest \
-vv \
--cov=litellm \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
-n 4 \
--timeout=300 \
--timeout_method=thread"
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml local_testing_part2_coverage.xml
mv .coverage local_testing_part2_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- local_testing_part2_coverage.xml
- local_testing_part2_coverage
langfuse_logging_unit_tests:
docker:
- image: cimg/python:3.11
@ -499,7 +639,6 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
@ -513,6 +652,7 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-xdist==3.6.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
# Run pytest and generate JUnit XML report
@ -575,8 +715,8 @@ jobs:
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_router_coverage.xml
mv .coverage litellm_router_coverage
mv coverage.xml litellm_router_unit_coverage.xml
mv .coverage litellm_router_unit_coverage
# Store test results
- store_test_results:
path: test-results
@ -584,8 +724,8 @@ jobs:
- persist_to_workspace:
root: .
paths:
- litellm_router_coverage.xml
- litellm_router_coverage
- litellm_router_unit_coverage.xml
- litellm_router_unit_coverage
litellm_security_tests:
machine:
image: ubuntu-2204:2023.10.1
@ -1152,8 +1292,8 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
pip install "mcp==1.10.1"
pip install "pydantic==2.11.0"
pip install "mcp==1.25.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@ -1556,8 +1696,8 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
pip install "pydantic==2.10.2"
pip install "mcp==1.10.1"
pip install "pydantic==2.11.0"
pip install "mcp==1.25.0"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
@ -1743,13 +1883,14 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/image_gen_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5
python -m pytest -vv tests/image_gen_tests -n 4 --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -1792,6 +1933,7 @@ jobs:
pip install "mlflow==2.17.2"
pip install "anthropic==0.52.0"
pip install "blockbuster==1.5.24"
pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
@ -1799,7 +1941,7 @@ jobs:
command: |
pwd
ls
python -m pytest -vv tests/logging_callback_tests --cov=litellm --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
python -m pytest -vv tests/logging_callback_tests --cov=litellm -n 4 --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -1915,7 +2057,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "tomli==2.2.1"
pip install "mcp==1.10.1"
pip install "mcp==1.25.0"
- run:
name: Run tests
command: |
@ -2192,6 +2334,8 @@ jobs:
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.100.1"
pip install "litellm[proxy]"
pip install "pytest-xdist==3.6.1"
- run:
name: Install dockerize
command: |
@ -2268,7 +2412,7 @@ jobs:
command: |
pwd
ls
python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
no_output_timeout: 120m
# Store test results
@ -3263,6 +3407,110 @@ jobs:
- store_test_results:
path: test-results
proxy_e2e_anthropic_messages_tests:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.10
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.10 -y
conda activate myenv
python --version
- run:
name: Install Dependencies
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
pip install "pytest==7.3.1"
pip install "pytest-asyncio==0.21.1"
pip install "boto3==1.36.0"
pip install "httpx==0.27.0"
pip install "claude-agent-sdk"
pip install -r requirements.txt
- run:
name: Install dockerize
command: |
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container with test config
command: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME="us-east-1" \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
- run:
name: Start outputting logs
command: docker logs -f my-app
background: true
- run:
name: Wait for app to be ready
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Run Claude Agent SDK E2E Tests
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
pwd
ls
python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
# Store test results
- store_test_results:
path: test-results
upload-coverage:
docker:
- image: cimg/python:3.9
@ -3284,7 +3532,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@ -3334,8 +3582,22 @@ jobs:
ls dist/
twine upload --verbose dist/*
else
echo "Version ${VERSION} of package is already published on PyPI. Skipping PyPI publish."
circleci step halt
echo "Version ${VERSION} of package is already published on PyPI."
# Check if corresponding Docker nightly image exists
NIGHTLY_TAG="v${VERSION}-nightly"
echo "Checking for Docker nightly image: litellm/litellm:${NIGHTLY_TAG}"
# Check Docker Hub for the nightly image
if curl -s "https://hub.docker.com/v2/repositories/litellm/litellm/tags/${NIGHTLY_TAG}" | grep -q "name"; then
echo "Docker nightly image ${NIGHTLY_TAG} exists. This release was already completed successfully."
echo "Skipping PyPI publish and continuing to ensure Docker images are up to date."
circleci step halt
else
echo "ERROR: PyPI package ${VERSION} exists but Docker nightly image ${NIGHTLY_TAG} does not exist!"
echo "This indicates an incomplete release. Please investigate."
exit 1
fi
fi
- run:
name: Trigger Github Action for new Docker Container + Trigger Load Testing
@ -3344,11 +3606,21 @@ jobs:
python3 -m pip install toml
VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
echo "LiteLLM Version ${VERSION}"
# Determine which branch to use for Docker build
if [[ "$CIRCLE_BRANCH" =~ ^litellm_release_day_.* ]]; then
BUILD_BRANCH="$CIRCLE_BRANCH"
echo "Using release branch: $BUILD_BRANCH"
else
BUILD_BRANCH="main"
echo "Using default branch: $BUILD_BRANCH"
fi
curl -X POST \
-H "Accept: application/vnd.github.v3+json" \
-H "Authorization: Bearer $GITHUB_TOKEN" \
"https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \
-d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
-d "{\"ref\":\"${BUILD_BRANCH}\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
echo "triggering load testing server for version ${VERSION} and commit ${CIRCLE_SHA1}"
curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}&release_type=nightly"
@ -3739,7 +4011,13 @@ workflows:
only:
- main
- /litellm_.*/
- local_testing:
- local_testing_part1:
filters:
branches:
only:
- main
- /litellm_.*/
- local_testing_part2:
filters:
branches:
only:
@ -3901,6 +4179,14 @@ workflows:
only:
- main
- /litellm_.*/
- proxy_e2e_anthropic_messages_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- llm_translation_testing:
filters:
branches:
@ -4044,7 +4330,8 @@ workflows:
- litellm_proxy_unit_testing_part2
- litellm_security_tests
- langfuse_logging_unit_tests
- local_testing
- local_testing_part1
- local_testing_part2
- litellm_assistants_api_testing
- auth_ui_unit_tests
- db_migration_disable_update_check:
@ -4084,10 +4371,12 @@ workflows:
branches:
only:
- main
- /litellm_release_day_.*/
- publish_to_pypi:
requires:
- mypy_linting
- local_testing
- local_testing_part1
- local_testing_part2
- build_and_test
- e2e_openai_endpoints
- test_bad_database_url

7
.circleci/requirements.txt
View file

@ -8,12 +8,13 @@ redis==5.2.1
redisvl==0.4.1
anthropic
orjson==3.10.12 # fast /embedding responses
pydantic==2.10.2
pydantic==2.11.0
google-cloud-aiplatform==1.43.0
google-cloud-iam==2.19.1
fastapi-sso==0.16.0
uvloop==0.21.0
mcp==1.10.1 # for MCP server
mcp==1.25.0 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
fastuuid==0.12.0
responses==0.25.7 # for proxy client tests
responses==0.25.7 # for proxy client tests
pytest-retry==1.6.3 # for automatic test retries

8
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View file

@ -9,6 +9,14 @@ body:
Thanks for taking the time to fill out this bug report!
**💡 Tip:** See our [Troubleshooting Guide](https://docs.litellm.ai/docs/troubleshoot) for what information to include.
- type: checkboxes
id: duplicate-check
attributes:
label: Check for existing issues
description: Please search to see if an issue already exists for the bug you encountered.
options:
- label: I have searched the existing issues and checked that my issue is not a duplicate.
required: true
- type: textarea
id: what-happened
attributes:

8
.github/ISSUE_TEMPLATE/feature_request.yml vendored
View file

@ -7,6 +7,14 @@ body:
attributes:
value: |
Thanks for making LiteLLM better!
- type: checkboxes
id: duplicate-check
attributes:
label: Check for existing issues
description: Please search to see if an issue already exists for the feature you are requesting.
options:
- label: I have searched the existing issues and checked that my issue is not a duplicate.
required: true
- type: textarea
id: the-feature
attributes:

29
.github/workflows/check_duplicate_issues.yml vendored Normal file
View file

@ -0,0 +1,29 @@
name: Check Duplicate Issues
on:
issues:
types: [opened, edited]
jobs:
check-duplicate:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Check for potential duplicates
uses: wow-actions/potential-duplicates@v1
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
label: potential-duplicate
threshold: 0.6
reaction: eyes
comment: |
**⚠️ Potential duplicate detected**
This issue appears similar to existing issue(s):
{{#issues}}
- [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
{{/issues}}
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.

4
.github/workflows/create_daily_staging_branch.yml vendored
View file

@ -2,7 +2,7 @@ name: Create Daily Staging Branch
on:
schedule:
- cron: '0 0 * * *' # Runs daily at midnight UTC
- cron: '0 0,12 * * *' # Runs every 12 hours at midnight and noon UTC
workflow_dispatch: # Allow manual trigger
jobs:
@ -24,7 +24,7 @@ jobs:
git config user.email "github-actions[bot]@users.noreply.github.com"
# Generate branch name with MM_DD_YYYY format
BRANCH_NAME="litellm_staging_$(date +'%m_%d_%Y')"
BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')"
echo "Creating branch: $BRANCH_NAME"
# Fetch all branches

66
.github/workflows/ghcr_deploy.yml vendored
View file

@ -320,72 +320,36 @@ jobs:
run: |
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
- name: Get LiteLLM Latest Tag
id: current_app_tag
shell: bash
run: |
LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0)
if [ -z "${LATEST_TAG}" ]; then
echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT
else
echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT
fi
- name: Get last published chart version
id: current_version
shell: bash
run: |
CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
if [ -z "${CHART_LIST}" ]; then
echo "current-version=1.0.0" | tee -a $GITHUB_OUTPUT
else
# Extract version and strip any prerelease suffix (e.g., 1.0.5-latest -> 1.0.5)
VERSION=$(printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print $2}' | tr -d " " | cut -d'-' -f1)
echo "current-version=${VERSION}" | tee -a $GITHUB_OUTPUT
fi
env:
HELM_EXPERIMENTAL_OCI: '1'
# Automatically update the helm chart version one "patch" level
- name: Bump release version
id: bump_version
uses: christian-draeger/increment-semantic-version@1.1.0
with:
current-version: ${{ steps.current_version.outputs.current-version || '1.0.0' }}
version-fragment: 'bug'
# Add suffix for non-stable releases (semantic versioning)
# Sync Helm chart version with LiteLLM release version (1-1 versioning)
# This allows users to easily map Helm chart versions to LiteLLM versions
# See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/
- name: Calculate chart and app versions
id: chart_version
shell: bash
run: |
BASE_VERSION="${{ steps.bump_version.outputs.next-version || '1.0.0' }}"
RELEASE_TYPE="${{ github.event.inputs.release_type }}"
INPUT_TAG="${{ github.event.inputs.tag }}"
RELEASE_TYPE="${{ github.event.inputs.release_type }}"
# Chart version (independent Helm chart versioning with release type suffix)
if [ "$RELEASE_TYPE" = "stable" ]; then
echo "version=${BASE_VERSION}" | tee -a $GITHUB_OUTPUT
else
echo "version=${BASE_VERSION}-${RELEASE_TYPE}" | tee -a $GITHUB_OUTPUT
# Chart version = LiteLLM version without 'v' prefix (Helm semver convention)
# v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1
CHART_VERSION="${INPUT_TAG#v}"
# Add suffix for 'latest' releases (rc already has suffix in tag)
if [ "$RELEASE_TYPE" = "latest" ]; then
CHART_VERSION="${CHART_VERSION}-latest"
fi
# App version (must match Docker tags)
# stable/rc releases: Docker creates main-{tag}, so use the tag
# latest/dev releases: Docker only creates main-{release_type}, so use release_type
if [ "$RELEASE_TYPE" = "stable" ] || [ "$RELEASE_TYPE" = "rc" ]; then
APP_VERSION="${INPUT_TAG}"
else
APP_VERSION="${RELEASE_TYPE}"
fi
# App version = Docker tag (keeps 'v' prefix to match Docker image tags)
APP_VERSION="${INPUT_TAG}"
echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
- uses: ./.github/actions/helm-oci-chart-releaser
with:
name: ${{ env.CHART_NAME }}
repository: ${{ env.REPO_OWNER }}
tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '1.0.0' }}
tag: ${{ steps.chart_version.outputs.version }}
app_version: ${{ steps.chart_version.outputs.app_version }}
path: deploy/charts/${{ env.CHART_NAME }}
registry: ${{ env.REGISTRY }}

42
.github/workflows/ghcr_helm_deploy.yml vendored
View file

@ -1,10 +1,12 @@
# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
# Standalone workflow to publish LiteLLM Helm Chart
# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release
name: Build, Publish LiteLLM Helm Chart. New Release
on:
workflow_dispatch:
inputs:
chartVersion:
description: "Update the helm chart's version to this"
tag:
description: "LiteLLM version tag (e.g., v1.81.0)"
required: true
# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
@ -31,24 +33,22 @@ jobs:
run: |
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
- name: Get LiteLLM Latest Tag
id: current_app_tag
uses: WyriHaximus/github-action-get-previous-tag@v1.3.0
- name: Get last published chart version
id: current_version
# Sync Helm chart version with LiteLLM release version (1-1 versioning)
- name: Calculate chart and app versions
id: chart_version
shell: bash
run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
env:
HELM_EXPERIMENTAL_OCI: '1'
run: |
INPUT_TAG="${{ github.event.inputs.tag }}"
# Automatically update the helm chart version one "patch" level
- name: Bump release version
id: bump_version
uses: christian-draeger/increment-semantic-version@1.1.0
with:
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
version-fragment: 'bug'
# Chart version = LiteLLM version without 'v' prefix
# v1.81.0 -> 1.81.0
CHART_VERSION="${INPUT_TAG#v}"
# App version = Docker tag (keeps 'v' prefix)
APP_VERSION="${INPUT_TAG}"
echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
- name: Lint helm chart
run: helm lint deploy/charts/litellm-helm
@ -57,8 +57,8 @@ jobs:
with:
name: litellm-helm
repository: ${{ env.REPO_OWNER }}
tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
tag: ${{ steps.chart_version.outputs.version }}
app_version: ${{ steps.chart_version.outputs.app_version }}
path: deploy/charts/litellm-helm
registry: ${{ env.REGISTRY }}
registry_username: ${{ github.actor }}

34
.github/workflows/label-component.yml vendored
View file

@ -80,3 +80,37 @@ jobs:
break;
}
}
// Check for 'claude code' keyword (can be applied alongside component labels)
if (/claude code/i.test(body)) {
const claudeLabel = {
name: 'claude code',
color: '7c3aed',
description: 'Issues related to Claude Code usage'
};
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: claudeLabel.name
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: claudeLabel.name,
color: claudeLabel.color,
description: claudeLabel.description
});
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [claudeLabel.name]
});
}

2
.github/workflows/test-linting.yml vendored
View file

@ -73,4 +73,4 @@ jobs:
- name: Check import safety
run: |
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)

2
.github/workflows/test-litellm.yml vendored
View file

@ -34,7 +34,7 @@ jobs:
poetry run pip install "google-genai==1.22.0"
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.18"
poetry run pip install "python-multipart==0.0.22"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |

4
.github/workflows/test-mcp.yml vendored
View file

@ -34,8 +34,8 @@ jobs:
poetry run pip install "pytest-cov==5.0.0"
poetry run pip install "pytest-asyncio==0.21.1"
poetry run pip install "respx==0.22.0"
poetry run pip install "pydantic==2.10.2"
poetry run pip install "mcp==1.10.1"
poetry run pip install "pydantic==2.11.0"
poetry run pip install "mcp==1.25.0"
poetry run pip install pytest-xdist
- name: Setup litellm-enterprise as local package

15
.github/workflows/test-model-map.yaml vendored Normal file
View file

@ -0,0 +1,15 @@
name: Validate model_prices_and_context_window.json
on:
pull_request:
branches: [ main ]
jobs:
validate-model-prices-json:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Validate model_prices_and_context_window.json
run: |
jq empty model_prices_and_context_window.json

10
.gitignore vendored
View file

@ -1,5 +1,6 @@
.python-version
.venv
.venv_policy_test
.env
.newenv
newenv/*
@ -59,10 +60,6 @@ litellm/proxy/_super_secret_config.yaml
litellm/proxy/myenv/bin/activate
litellm/proxy/myenv/bin/Activate.ps1
myenv/*
litellm/proxy/_experimental/out/_next/
litellm/proxy/_experimental/out/404/index.html
litellm/proxy/_experimental/out/model_hub/index.html
litellm/proxy/_experimental/out/onboarding/index.html
litellm/tests/log.txt
litellm/tests/langfuse.log
litellm/tests/langfuse.log
@ -75,9 +72,6 @@ tests/local_testing/log.txt
litellm/proxy/_new_new_secret_config.yaml
litellm/proxy/custom_guardrail.py
.mypy_cache/*
litellm/proxy/_experimental/out/404.html
litellm/proxy/_experimental/out/404.html
litellm/proxy/_experimental/out/model_hub.html
.mypy_cache/*
litellm/proxy/application.log
tests/llm_translation/vertex_test_account.json
@ -99,9 +93,9 @@ litellm_config.yaml
litellm/proxy/to_delete_loadtest_work/*
update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
litellm/proxy/_experimental/out/guardrails/index.html
scripts/test_vertex_ai_search.py
LAZY_LOADING_IMPROVEMENTS.md
STABILIZATION_TODO.md
**/test-results
**/playwright-report
**/*.storageState.json

12
.trivyignore Normal file
View file

@ -0,0 +1,12 @@
# LiteLLM Trivy Ignore File
# CVEs listed here are temporarily allowlisted pending fixes
# Next.js vulnerabilities in UI dashboard (next@14.2.35)
# Allowlisted: 2026-01-31, 7-day fix timeline
# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+
# HIGH: DoS via request deserialization
GHSA-h25m-26qc-wcjf
# MEDIUM: Image Optimizer DoS
CVE-2025-59471

8
AGENTS.md
View file

@ -51,12 +51,14 @@ LiteLLM is a unified interface for 100+ LLMs that:
### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
1. **Use Common Components as much as possible**:
1. **Tremor is DEPRECATED, do not use Tremor components in new features/changes**
- The only exception is the Tremor Table component and its required Tremor Table sub components.
2. **Use Common Components as much as possible**:
- These are usually defined in the `common_components` directory
- Use these components as much as possible and avoid building new components unless needed
- Tremor components are deprecated; prefer using Ant Design (AntD) as much as possible
2. **Testing**:
3. **Testing**:
- The codebase uses **Vitest** and **React Testing Library**
- **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
- **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)

127
ARCHITECTURE.md
View file

@ -28,16 +28,25 @@ sequenceDiagram
participant Client
participant ProxyServer as proxy/proxy_server.py
participant Auth as proxy/auth/user_api_key_auth.py
participant Redis as Redis Cache
participant Hooks as proxy/hooks/
participant Router as router.py
participant Main as main.py
participant Main as main.py + utils.py
participant Handler as llms/custom_httpx/llm_http_handler.py
participant Transform as llms/{provider}/chat/transformation.py
participant Provider as LLM Provider API
participant CostCalc as cost_calculator.py
participant LoggingObj as litellm_logging.py
participant DBWriter as db/db_spend_update_writer.py
participant Postgres as PostgreSQL
%% Request Flow
Client->>ProxyServer: POST /v1/chat/completions
ProxyServer->>Auth: user_api_key_auth()
Auth->>Redis: Check API key cache
Redis-->>Auth: Key info + spend limits
ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter
Hooks->>Redis: Check/increment rate limit counters
ProxyServer->>Router: route_request()
Router->>Main: litellm.acompletion()
Main->>Handler: BaseLLMHTTPHandler.completion()
@ -45,8 +54,25 @@ sequenceDiagram
Handler->>Provider: HTTP Request
Provider-->>Handler: Response
Handler->>Transform: ProviderConfig.transform_response()
Handler-->>Hooks: async_log_success_event()
Handler-->>Client: ModelResponse
Transform-->>Handler: ModelResponse
Handler-->>Main: ModelResponse
%% Cost Attribution (in utils.py wrapper)
Main->>LoggingObj: update_response_metadata()
LoggingObj->>CostCalc: _response_cost_calculator()
CostCalc->>CostCalc: completion_cost(tokens × price)
CostCalc-->>LoggingObj: response_cost
LoggingObj-->>Main: Set response._hidden_params["response_cost"]
Main-->>ProxyServer: ModelResponse (with cost in _hidden_params)
%% Response Headers + Async Logging
ProxyServer->>ProxyServer: Extract cost from hidden_params
ProxyServer->>LoggingObj: async_success_handler()
LoggingObj->>Hooks: async_log_success_event()
Hooks->>DBWriter: update_database(response_cost)
DBWriter->>Redis: Queue spend increment
DBWriter->>Postgres: Batch write spend logs (async)
ProxyServer-->>Client: ModelResponse + x-litellm-response-cost header
```
### Proxy Components
@ -75,11 +101,19 @@ graph TD
Main["main.py"]
end
subgraph "Infrastructure"
DualCache["DualCache<br/>(in-memory + Redis)"]
Postgres["PostgreSQL<br/>(keys, teams, spend logs)"]
end
Client --> Endpoint
Endpoint --> Auth
Auth --> DualCache
DualCache -.->|cache miss| Postgres
Auth --> PreCall
PreCall --> RouteRequest
RouteRequest --> Router
Router --> DualCache
Router --> Main
Main --> Client
```
@ -119,6 +153,93 @@ graph TD
To add a new proxy hook, implement `CustomLogger` and register in `PROXY_HOOKS`.
### Infrastructure Components
The AI Gateway uses external infrastructure for persistence and caching:
```mermaid
graph LR
subgraph "AI Gateway (proxy/)"
Proxy["proxy_server.py"]
Auth["auth/user_api_key_auth.py"]
DBWriter["db/db_spend_update_writer.py<br/>DBSpendUpdateWriter"]
InternalCache["utils.py<br/>InternalUsageCache"]
CostCallback["hooks/proxy_track_cost_callback.py<br/>_ProxyDBLogger"]
Scheduler["APScheduler<br/>ProxyStartupEvent"]
end
subgraph "SDK (litellm/)"
Router["router.py<br/>Router.cache (DualCache)"]
LLMCache["caching/caching_handler.py<br/>LLMCachingHandler"]
CacheClass["caching/caching.py<br/>Cache"]
end
subgraph "Redis (caching/redis_cache.py)"
RateLimit["Rate Limit Counters"]
SpendQueue["Spend Increment Queue"]
KeyCache["API Key Cache"]
TPM_RPM["TPM/RPM Tracking"]
Cooldowns["Deployment Cooldowns"]
LLMResponseCache["LLM Response Cache"]
end
subgraph "PostgreSQL (proxy/schema.prisma)"
Keys["LiteLLM_VerificationToken"]
Teams["LiteLLM_TeamTable"]
SpendLogs["LiteLLM_SpendLogs"]
Users["LiteLLM_UserTable"]
end
Auth --> InternalCache
InternalCache --> KeyCache
InternalCache -.->|cache miss| Keys
InternalCache --> RateLimit
Router --> TPM_RPM
Router --> Cooldowns
LLMCache --> CacheClass
CacheClass --> LLMResponseCache
CostCallback --> DBWriter
DBWriter --> SpendQueue
DBWriter --> SpendLogs
Scheduler --> SpendLogs
Scheduler --> Keys
```
| Component | Purpose | Key Files/Classes |
|-----------|---------|-------------------|
| **Redis** | Rate limiting, API key caching, TPM/RPM tracking, cooldowns, LLM response caching, spend queuing | `caching/redis_cache.py` (`RedisCache`), `caching/dual_cache.py` (`DualCache`) |
| **PostgreSQL** | API keys, teams, users, spend logs | `proxy/utils.py` (`PrismaClient`), `proxy/schema.prisma` |
| **InternalUsageCache** | Proxy-level cache for rate limits + API keys (in-memory + Redis) | `proxy/utils.py` (`InternalUsageCache`) |
| **Router.cache** | TPM/RPM tracking, deployment cooldowns, client caching (in-memory + Redis) | `router.py` (`Router.cache: DualCache`) |
| **LLMCachingHandler** | SDK-level LLM response/embedding caching | `caching/caching_handler.py` (`LLMCachingHandler`), `caching/caching.py` (`Cache`) |
| **DBSpendUpdateWriter** | Batches spend updates to reduce DB writes | `proxy/db/db_spend_update_writer.py` (`DBSpendUpdateWriter`) |
| **Cost Tracking** | Calculates and logs response costs | `proxy/hooks/proxy_track_cost_callback.py` (`_ProxyDBLogger`) |
**Background Jobs** (APScheduler, initialized in `proxy/proxy_server.py` → `ProxyStartupEvent.initialize_scheduled_background_jobs()`):
| Job | Interval | Purpose | Key Files |
|-----|----------|---------|-----------|
| `update_spend` | 60s | Batch write spend logs to PostgreSQL | `proxy/db/db_spend_update_writer.py` |
| `reset_budget` | 10-12min | Reset budgets for keys/users/teams | `proxy/management_helpers/budget_reset_job.py` |
| `add_deployment` | 10s | Sync new model deployments from DB | `proxy/proxy_server.py` (`ProxyConfig`) |
| `cleanup_old_spend_logs` | cron/interval | Delete old spend logs | `proxy/management_helpers/spend_log_cleanup.py` |
| `check_batch_cost` | 30min | Calculate costs for batch jobs | `proxy/management_helpers/check_batch_cost_job.py` |
| `check_responses_cost` | 30min | Calculate costs for responses API | `proxy/management_helpers/check_responses_cost_job.py` |
| `process_rotations` | 1hr | Auto-rotate API keys | `proxy/management_helpers/key_rotation_manager.py` |
| `_run_background_health_check` | continuous | Health check model deployments | `proxy/proxy_server.py` |
| `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
| `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
**Cost Attribution Flow:**
1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes
2. `update_response_metadata()` (`llm_response_utils/response_metadata.py`) is called
3. `logging_obj._response_cost_calculator()` (`litellm_logging.py`) calculates cost via `litellm.completion_cost()` (`cost_calculator.py`)
4. Cost is stored in `response._hidden_params["response_cost"]`
5. `proxy/common_request_processing.py` extracts cost from `hidden_params` and adds to response headers (`x-litellm-response-cost`)
6. `logging_obj.async_success_handler()` triggers callbacks including `_ProxyDBLogger.async_log_success_event()`
7. `DBSpendUpdateWriter.update_database()` queues spend increments to Redis
8. Background job `update_spend` flushes queued spend to PostgreSQL every 60s
---
## 2. SDK Request Flow

9
Dockerfile
View file

@ -46,8 +46,9 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
# Ensure runtime stage runs as root
USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@latest
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -69,8 +70,8 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Generate prisma client
RUN prisma generate
# Generate prisma client using the correct schema
RUN prisma generate --schema=./litellm/proxy/schema.prisma
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh

17
README.md
View file

@ -258,6 +258,19 @@ LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https:
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
## OSS Adopters
<table>
<tr>
<td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
<td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
<td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
<td><h2>Netflix</h2></td>
<td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
</tr>
</table>
## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
@ -374,7 +387,9 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
1. (In root) create virtual environment `python -m venv .venv`
2. Activate virtual environment `source .venv/bin/activate`
3. Install dependencies `pip install -e ".[all]"`
4. Start proxy backend `python litellm/proxy_cli.py`
4. `pip install prisma`
5. `prisma generate`
6. Start proxy backend `python litellm/proxy/proxy_cli.py`
### Frontend
1. Navigate to `ui/litellm-dashboard`

4
batch_small.jsonl
View file

@ -1,4 +0,0 @@
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello, how are you?"}]}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "What is the weather today?"}]}}
{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Tell me a short joke"}]}}

4
ci_cd/.grype.yaml
View file

@ -1,3 +1,3 @@
ignore:
- vulnerability: CVE-2019-1010022
reason: no fixed glibc package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
- vulnerability: CVE-2026-22184
reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists

24
ci_cd/security_scans.sh
View file

@ -81,10 +81,10 @@ run_trivy_scans() {
echo "Running Trivy scans..."
echo "Scanning LiteLLM Docs..."
trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
echo "Scanning LiteLLM UI..."
trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
echo "Trivy scans completed successfully"
}
@ -129,11 +129,31 @@ run_grype_scans() {
"CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
"CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
"CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
"CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet
"CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
"CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
"CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
"CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
"GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
"GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
"GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
"CVE-2025-59465" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-55131" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59466" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-55130" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59467" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2026-21637" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-15281" # No fix available yet
"CVE-2026-0865" # No fix available yet
"CVE-2025-15282" # No fix available yet
"CVE-2026-0672" # No fix available yet
"CVE-2025-15366" # No fix available yet
"CVE-2025-15367" # No fix available yet
"CVE-2025-12781" # No fix available yet
"CVE-2025-11468" # No fix available yet
)
# Build JSON array of allowlisted CVE IDs for jq

295
cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md Normal file
View file

@ -0,0 +1,295 @@
# Claude Code with LiteLLM Quickstart
This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code.
> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
## Video Walkthrough
Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56
## Prerequisites
- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
- API keys for your chosen providers
## Installation
First, install LiteLLM with proxy support:
```bash
pip install 'litellm[proxy]'
```
## Step 1: Setup config.yaml
Create a secure configuration using environment variables:
```yaml
model_list:
# Claude models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
Set your environment variables:
```bash
export ANTHROPIC_API_KEY="your-anthropic-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
## Step 2: Start Proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
## Step 3: Verify Setup
Test that your proxy is working correctly:
```bash
curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "claude-3-5-sonnet-20241022",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
## Step 4: Configure Claude Code
### Method 1: Unified Endpoint (Recommended)
Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI.
### Method 2: Provider-specific Pass-through Endpoint
Alternatively, use the Anthropic pass-through endpoint:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
## Step 5: Use Claude Code
### Choosing Your Model
You have two options for specifying which model Claude Code uses:
#### Option 1: Command Line / Session Model Selection
Specify the model directly when starting Claude Code or during a session:
```bash
# Specify model at startup
claude --model claude-3-5-sonnet-20241022
# Or change model during a session
/model claude-3-5-haiku-20241022
```
This method uses the exact model you specify.
#### Option 2: Environment Variables
Configure default models using environment variables:
```bash
# Tell Claude Code which models to use by default
export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
export ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-3-5-20240229
claude # Will use the models specified above
```
**Note:** Claude Code may cache the model from a previous session. If environment variables don't take effect, use Option 1 to explicitly set the model.
**Important:** The `model_name` in your LiteLLM config must match what Claude Code requests (either from env vars or command line).
### Using 1M Context Window
Claude Code supports extended context (1 million tokens) using the `[1m]` suffix with Claude 4+ models:
```bash
# Use Sonnet 4.5 with 1M context (requires quotes for shell)
claude --model 'claude-sonnet-4-5-20250929[1m]'
# Inside a Claude Code session (no quotes needed)
/model claude-sonnet-4-5-20250929[1m]
```
**Important:** When using `--model` with `[1m]` in the shell, you must use quotes to prevent the shell from interpreting the brackets.
Alternatively, set as default with environment variables:
```bash
export ANTHROPIC_DEFAULT_SONNET_MODEL='claude-sonnet-4-5-20250929[1m]'
claude
```
**How it works:**
- Claude Code strips the `[1m]` suffix before sending to LiteLLM
- Claude Code automatically adds the header `anthropic-beta: context-1m-2025-08-07`
- Your LiteLLM config should **NOT** include `[1m]` in model names
**Verify 1M context is active:**
```bash
/context
# Should show: 21k/1000k tokens (2%)
```
**Pricing:** Models using 1M context have different pricing. Input tokens above 200k are charged at a higher rate.
## Troubleshooting
Common issues and solutions:
**Claude Code not connecting:**
- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
- Check that `ANTHROPIC_BASE_URL` is set correctly
- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Authentication errors:**
- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
- Check that your API keys are valid and have sufficient credits
- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Model not found:**
- Check what model Claude Code is requesting in LiteLLM logs
- Ensure your `config.yaml` has a matching `model_name` entry
- If using environment variables, verify they're set: `echo $ANTHROPIC_DEFAULT_SONNET_MODEL`
**1M context not working (showing 200k instead of 1000k):**
- Verify you're using the `[1m]` suffix: `/model your-model-name[1m]`
- Check LiteLLM logs for the header `context-1m-2025-08-07` in the request
- Ensure your model supports 1M context (only certain Claude models do)
- Your LiteLLM config should **NOT** include `[1m]` in the `model_name`
## Using Multiple Models and Providers
You can configure LiteLLM to route to any supported provider. Here's an example with multiple providers:
```yaml
model_list:
# OpenAI models
- model_name: codex-mini
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: o3-pro
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
# Anthropic models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
**Note:** The `model_name` can be anything you choose. Claude Code will request whatever model you specify (via env vars or command line), and LiteLLM will route to the `model` configured in `litellm_params`.
Switch between models seamlessly:
```bash
# Use environment variables to set defaults
export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
# Or specify directly
claude --model claude-3-5-sonnet-20241022 # Complex reasoning
claude --model claude-3-5-haiku-20241022 # Fast responses
claude --model claude-bedrock # Bedrock deployment
```
## Default Models Used by Claude Code
If you **don't** set environment variables, Claude Code uses these default model names:
| Purpose | Default Model Name (v2.1.14) |
|---------|------------------------------|
| Main model | `claude-sonnet-4-5-20250929` |
| Light tasks (subagents, summaries) | `claude-haiku-4-5-20251001` |
| Planning mode | `claude-opus-4-5-20251101` |
Your LiteLLM config should include these model names if you want Claude Code to work without setting environment variables:
```yaml
model_list:
- model_name: claude-sonnet-4-5-20250929
litellm_params:
# Can be any provider - Anthropic, Bedrock, Vertex AI, etc.
model: anthropic/claude-sonnet-4-5-20250929
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-haiku-4-5-20251001
litellm_params:
model: anthropic/claude-haiku-4-5-20251001
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-opus-4-5-20251101
litellm_params:
model: anthropic/claude-opus-4-5-20251101
api_key: os.environ/ANTHROPIC_API_KEY
```
**Warning:** These default model names may change with new Claude Code versions. Check LiteLLM proxy logs for "model not found" errors to identify what Claude Code is requesting.
## Additional Resources
- [LiteLLM Documentation](https://docs.litellm.ai/)
- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)

134
cookbook/ai_coding_tool_guides/index.json Normal file
View file

@ -0,0 +1,134 @@
[{
"title": "Claude Code Quickstart",
"description": "This is a quickstart guide to using Claude Code with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api",
"date": "2026-01-15",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM"
]
},
{
"title": "Claude Code with MCPs",
"description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_mcp",
"date": "2026-01-15",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"MCP"
]
},
{
"title": "Claude Code with Non-Anthropic Models",
"description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"OpenAI",
"Gemini"
]
},
{
"title": "Cursor Quickstart",
"description": "This is a quickstart guide to using Cursor with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/cursor_integration",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Cursor",
"LiteLLM",
"Quickstart"
]
},
{
"title": "Github Copilot Quickstart",
"description": "This is a quickstart guide to using Github Copilot with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Github Copilot",
"LiteLLM",
"Quickstart"
]
},
{
"title": "LiteLLM Gemini CLI Quickstart",
"description": "This is a quickstart guide to using LiteLLM Gemini CLI.",
"url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Gemini CLI",
"Gemini",
"LiteLLM",
"Quickstart"
]
},
{
"title": "OpenAI Codex CLI Quickstart",
"description": "This is a quickstart guide to using OpenAI Codex CLI.",
"url": "https://docs.litellm.ai/docs/tutorials/openai_codex",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"OpenAI Codex CLI",
"OpenAI",
"LiteLLM",
"Quickstart"
]
},
{
"title": "OpenWebUI Quickstart",
"description": "This is a quickstart guide to using OpenWebUI with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/openweb_ui",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"OpenWebUI",
"LiteLLM",
"Quickstart"
]
},
{
"title": "AI Coding Tool Usage Tracking",
"description": "This is a guide to tracking usage for AI coding tools monitor the use of Claude Code , Google Antigravity, OpenAI Codex, Roo Code etc. through LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/cost_tracking_coding",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"Gemini CLI",
"OpenAI Codex",
"LiteLLM"
]
},
{
"title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)",
"description": "This is a guide for using Web Search with Claude Code via LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"Web Search"
]
},
{
"title": "Track Claude Code Usage per user via Custom Headers",
"description": "This is a guide for tracking claude code user usage by passing a customer ID header.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_code_customer_tracking",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM"
]
}]

144
cookbook/anthropic_agent_sdk/README.md Normal file
View file

@ -0,0 +1,144 @@
# Claude Agent SDK with LiteLLM Gateway
A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK.
## Quick Start
### 1. Install dependencies
```bash
pip install anthropic claude-agent-sdk litellm
```
### 2. Start LiteLLM proxy
```bash
# Simple start with Claude
litellm --model claude-sonnet-4-20250514
# Or with a config file
litellm --config config.yaml
```
### 3. Run the chat
**Basic Agent (no MCP):**
```bash
python main.py
```
**Agent with MCP (DeepWiki2 for research):**
```bash
python agent_with_mcp.py
```
If MCP connection fails, you can disable it:
```bash
USE_MCP=false python agent_with_mcp.py
```
That's it! You can now chat with the agent in your terminal.
### Chat Commands
While chatting, you can use these commands:
- `models` - List all available models (fetched from your LiteLLM proxy)
- `model` - Switch to a different model
- `clear` - Start a new conversation
- `quit` or `exit` - End the chat
The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured.
## Configuration
Set these environment variables if needed:
```bash
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
export LITELLM_MODEL="bedrock-claude-sonnet-4.5"
```
Or just use the defaults - it'll connect to `http://localhost:4000` by default.
## Files
- `main.py` - Basic interactive agent without MCP
- `agent_with_mcp.py` - Agent with MCP server integration (DeepWiki2)
- `common.py` - Shared utilities and functions
- `config.example.yaml` - Example LiteLLM configuration
- `requirements.txt` - Python dependencies
## Example Config File
If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`):
```yaml
model_list:
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
```
Then start LiteLLM with: `litellm --config config.yaml`
## How It Works
The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
```python
# Point to LiteLLM gateway (not Anthropic)
os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key
# Use any model configured in LiteLLM
options = ClaudeAgentOptions(
model="bedrock-claude-sonnet-4", # or gpt-4, or anything else
system_prompt="You are a helpful assistant.",
max_turns=50,
)
```
Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically.
## Why Use This?
- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc.
- **Cost tracking**: LiteLLM tracks spending across all your agent conversations
- **Rate limiting**: Set budgets and limits on your agent usage
- **Load balancing**: Distribute requests across multiple API keys or regions
- **Fallbacks**: Automatically retry with a different model if one fails
## Troubleshooting
**Connection errors?**
- Make sure LiteLLM is running: `litellm --model your-model`
- Check the URL is correct (default: `http://localhost:4000`)
**Authentication errors?**
- Verify your LiteLLM API key is correct
- Make sure the model is configured in your LiteLLM setup
**Model not found?**
- Check the model name matches what's in your LiteLLM config
- Run `litellm --model your-model` to test it works
**Agent with MCP stuck or failing?**
- The MCP server might not be available at `http://localhost:4000/mcp/deepwiki2`
- Try disabling MCP: `USE_MCP=false python agent_with_mcp.py`
- Or use the basic agent: `python main.py`
## Learn More
- [LiteLLM Docs](https://docs.litellm.ai/)
- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk)
- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start)

140
cookbook/anthropic_agent_sdk/agent_with_mcp.py Normal file
View file

@ -0,0 +1,140 @@
"""
Interactive Claude Agent SDK CLI with MCP Support
This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy,
with MCP (Model Context Protocol) server integration for enhanced capabilities.
"""
import asyncio
import os
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
from common import (
Config,
fetch_available_models,
setup_litellm_env,
print_header,
handle_model_list,
handle_model_switch,
stream_response,
)
async def interactive_chat_with_mcp():
"""
Interactive CLI chat with the agent and MCP server
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
current_model = config.LITELLM_MODEL
# MCP server configuration
mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
if not use_mcp:
print("⚠️ MCP disabled via USE_MCP=false")
print_header(litellm_base_url, current_model, has_mcp=use_mcp)
while True:
# Configure agent options
if use_mcp:
try:
# Try with MCP server (HTTP transport)
# Using McpHttpServerConfig format from Agent SDK
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant with access to DeepWiki for research. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
mcp_servers={
"deepwiki2": {
"type": "http",
"url": mcp_server_url,
"headers": {
"Authorization": f"Bearer {config.LITELLM_API_KEY}"
}
}
},
)
except Exception as e:
print(f"⚠️ Warning: Could not configure MCP server: {e}")
print("Continuing without MCP...\n")
use_mcp = False
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
else:
# Without MCP
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
# Create agent client
try:
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
user_input = input("\n👤 You: ").strip()
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)
except Exception as e:
print(f"\n❌ Error creating agent client: {e}")
print("This might be an MCP configuration issue. Try running without MCP:")
print(" USE_MCP=false python agent_with_mcp.py")
print("\nOr use the basic agent:")
print(" python main.py")
return
def main():
"""Run interactive chat with MCP"""
try:
asyncio.run(interactive_chat_with_mcp())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
if __name__ == "__main__":
main()

160
cookbook/anthropic_agent_sdk/common.py Normal file
View file

@ -0,0 +1,160 @@
"""
Common utilities for Claude Agent SDK examples
"""
import os
import httpx
class Config:
"""Configuration for LiteLLM Gateway connection"""
# LiteLLM proxy URL (default to local instance)
LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
# LiteLLM API key (master key or virtual key)
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
# Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
"""
Fetch available models from LiteLLM proxy /models endpoint
"""
try:
async with httpx.AsyncClient() as client:
response = await client.get(
f"{base_url}/models",
headers={"Authorization": f"Bearer {api_key}"},
timeout=10.0
)
response.raise_for_status()
data = response.json()
return [model["id"] for model in data.get("data", [])]
except Exception as e:
print(f"⚠️ Warning: Could not fetch models from proxy: {e}")
print("Using default model list...")
# Fallback to default models
return [
"bedrock-claude-sonnet-3.5",
"bedrock-claude-sonnet-4",
"bedrock-claude-sonnet-4.5",
"bedrock-claude-opus-4.5",
"bedrock-nova-premier",
]
def setup_litellm_env(config: Config):
"""
Configure environment variables to point Agent SDK to LiteLLM
"""
litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
return litellm_base_url
def print_header(base_url: str, current_model: str, has_mcp: bool = False):
"""
Print the chat header
"""
mcp_indicator = " + MCP" if has_mcp else ""
print("=" * 70)
print(f"🤖 Claude Agent SDK with LiteLLM Gateway{mcp_indicator} - Interactive Chat")
print("=" * 70)
print(f"🚀 Connected to: {base_url}")
print(f"📦 Current model: {current_model}")
if has_mcp:
print("🔌 MCP: deepwiki2 enabled")
print("\nType your messages below. Commands:")
print(" - 'quit' or 'exit' to end the conversation")
print(" - 'clear' to start a new conversation")
print(" - 'model' to switch models")
print(" - 'models' to list available models")
print("=" * 70)
print()
def handle_model_list(available_models: list[str], current_model: str):
"""
Display available models
"""
print("\n📋 Available models:")
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
"""
Handle model switching
Returns:
tuple: (new_model, should_restart_conversation)
"""
print("\n📋 Select a model:")
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
try:
choice = input("\nEnter number (or press Enter to cancel): ").strip()
if choice:
idx = int(choice) - 1
if 0 <= idx < len(available_models):
new_model = available_models[idx]
print(f"\n✅ Switched to: {new_model}")
print("🔄 Starting new conversation with new model...\n")
return new_model, True
else:
print("❌ Invalid choice")
except (ValueError, IndexError):
print("❌ Invalid input")
return current_model, False
async def stream_response(client, user_input: str):
"""
Stream response from the agent
"""
print("\n🤖 Assistant: ", end='', flush=True)
try:
await client.query(user_input)
# Show loading indicator
print("⏳ thinking...", end='', flush=True)
# Stream the response
first_chunk = True
async for msg in client.receive_response():
# Clear loading indicator on first message
if first_chunk:
print("\r🤖 Assistant: ", end='', flush=True)
first_chunk = False
# Handle different message types
if hasattr(msg, 'type'):
if msg.type == 'content_block_delta':
# Streaming text delta
if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
print(msg.delta.text, end='', flush=True)
elif msg.type == 'content_block_start':
# Start of content block
if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
print(msg.content_block.text, end='', flush=True)
# Fallback to original content handling
if hasattr(msg, 'content'):
for content_block in msg.content:
if hasattr(content_block, 'text'):
print(content_block.text, end='', flush=True)
print() # New line after response
except Exception as e:
print(f"\r\n❌ Error: {e}")
print("Please check your LiteLLM gateway is running and configured correctly.")

25
cookbook/anthropic_agent_sdk/config.example.yaml Normal file
View file

@ -0,0 +1,25 @@
model_list:
- model_name: bedrock-claude-sonnet-3.5
litellm_params:
model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-opus-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-nova-premier
litellm_params:
model: "bedrock/amazon.nova-premier-v1:0"
aws_region_name: "us-east-1"

95
cookbook/anthropic_agent_sdk/main.py Normal file
View file

@ -0,0 +1,95 @@
"""
Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway
This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy.
LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.)
through the Claude Agent SDK by pointing it to the LiteLLM gateway.
"""
import asyncio
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
from common import (
Config,
fetch_available_models,
setup_litellm_env,
print_header,
handle_model_list,
handle_model_switch,
stream_response,
)
async def interactive_chat():
"""
Interactive CLI chat with the agent
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
current_model = config.LITELLM_MODEL
print_header(litellm_base_url, current_model)
while True:
# Configure agent options for each conversation
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
# Create agent client
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
user_input = input("\n👤 You: ").strip()
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)
def main():
"""Run interactive chat"""
try:
asyncio.run(interactive_chat())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
if __name__ == "__main__":
main()

2
cookbook/anthropic_agent_sdk/requirements.txt Normal file
View file

@ -0,0 +1,2 @@
claude-agent-sdk
httpx>=0.27.0

2
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 1.0.0
version: 1.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

9
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -10,7 +10,7 @@ metadata:
{{- toYaml .Values.deploymentLabels | nindent 4 }}
{{- end }}
spec:
{{- if not .Values.autoscaling.enabled }}
{{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
replicas: {{ .Values.replicaCount }}
{{- end }}
selector:
@ -38,6 +38,10 @@ spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- with .Values.extraInitContainers }}
initContainers:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: {{ include "litellm.name" . }}
securityContext:
@ -170,7 +174,8 @@ spec:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: litellm-config
mountPath: /etc/litellm/
mountPath: /etc/litellm/config.yaml
subPath: config.yaml
{{ if .Values.securityContext.readOnlyRootFilesystem }}
- name: tmp
mountPath: /tmp

37
deploy/charts/litellm-helm/templates/keda.yaml Normal file
View file

@ -0,0 +1,37 @@
{{- if and .Values.keda.enabled (not .Values.autoscaling.enabled) }}
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- if .Values.keda.scaledObject.annotations }}
annotations: {{ toYaml .Values.keda.scaledObject.annotations | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
name: {{ include "litellm.fullname" . }}
pollingInterval: {{ .Values.keda.pollingInterval }}
cooldownPeriod: {{ .Values.keda.cooldownPeriod }}
minReplicaCount: {{ .Values.keda.minReplicas }}
maxReplicaCount: {{ .Values.keda.maxReplicas }}
{{- with .Values.keda.fallback }}
fallback:
failureThreshold: {{ .failureThreshold | default 3 }}
replicas: {{ .replicas | default $.Values.keda.maxReplicas }}
{{- end }}
triggers:
{{- with .Values.keda.triggers }}
{{- toYaml . | nindent 2 }}
{{- end }}
advanced:
restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }}
{{- if .Values.keda.behavior }}
horizontalPodAutoscalerConfig:
behavior:
{{- with .Values.keda.behavior }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- end }}

4
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -35,6 +35,10 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.migrationJob.extraInitContainers }}
initContainers:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: prisma-migrations
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}"

3
deploy/charts/litellm-helm/tests/deployment_tests.yaml
View file

@ -136,7 +136,8 @@ tests:
path: spec.template.spec.containers[0].volumeMounts
content:
name: litellm-config
mountPath: /etc/litellm/
mountPath: /etc/litellm/config.yaml
subPath: config.yaml
- it: should work with lifecycle hooks
template: deployment.yaml
set:

43
deploy/charts/litellm-helm/values.yaml
View file

@ -156,6 +156,40 @@ autoscaling:
targetCPUUtilizationPercentage: 80
# targetMemoryUtilizationPercentage: 80
# Autoscaling with keda is mutually exclusive with hpa
keda:
enabled: false
minReplicas: 1
maxReplicas: 100
pollingInterval: 30
cooldownPeriod: 300
# fallback:
# failureThreshold: 3
# replicas: 11
restoreToOriginalReplicaCount: false
scaledObject:
annotations: {}
triggers: []
# - type: prometheus
# metadata:
# serverAddress: http://<prometheus-host>:9090
# metricName: http_requests_total
# threshold: '100'
# query: sum(rate(http_requests_total{deployment="my-deployment"}[2m]))
behavior: {}
# scaleDown:
# stabilizationWindowSeconds: 300
# policies:
# - type: Pods
# value: 1
# periodSeconds: 180
# scaleUp:
# stabilizationWindowSeconds: 300
# policies:
# - type: Pods
# value: 2
# periodSeconds: 60
# Additional volumes on the output Deployment definition.
volumes: []
# - name: foo
@ -200,6 +234,14 @@ db:
# instance. See the "postgresql" top level key for additional configuration.
deployStandalone: true
# Lifecycle hooks for the LiteLLM container
# Example:
# lifecycle:
# preStop:
# exec:
# command: ["/bin/sh", "-c", "sleep 10"]
lifecycle: {}
# Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored
# otherwise)
postgresql:
@ -239,6 +281,7 @@ migrationJob:
# cpu: 100m
# memory: 100Mi
extraContainers: []
extraInitContainers: []
# Hook configuration
hooks:

3
docker/Dockerfile.custom_ui
View file

@ -5,7 +5,8 @@ FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
WORKDIR /app
# Install Node.js and npm (adjust version as needed)
RUN apt-get update && apt-get install -y nodejs npm
RUN apt-get update && apt-get install -y nodejs npm && \
npm install -g npm@latest tar@latest
# Copy the UI source into the container
COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard

3
docker/Dockerfile.database
View file

@ -49,7 +49,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@latest
WORKDIR /app
# Copy the current directory contents into the container at /app

3
docker/Dockerfile.dev
View file

@ -61,7 +61,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libatomic1 \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@latest tar@latest
WORKDIR /app

16
docker/Dockerfile.health_check Normal file
View file

@ -0,0 +1,16 @@
FROM python:3.11-slim
WORKDIR /app
# Copy health check script and requirements
COPY scripts/health_check/health_check_client.py /app/health_check_client.py
COPY scripts/health_check/health_check_requirements.txt /app/requirements.txt
# Install dependencies
RUN pip install --no-cache-dir -r requirements.txt
# Make script executable
RUN chmod +x /app/health_check_client.py
# Set entrypoint
ENTRYPOINT ["python", "/app/health_check_client.py"]

10
docker/Dockerfile.non_root
View file

@ -15,6 +15,7 @@ USER root
RUN for i in 1 2 3; do \
apk add --no-cache \
python3 \
python3-dev \
py3-pip \
clang \
llvm \
@ -103,7 +104,8 @@ RUN for i in 1 2 3; do \
done \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done
done \
&& npm install -g npm@latest tar@latest
# Copy artifacts from builder
COPY --from=builder /app/requirements.txt /app/requirements.txt
@ -169,12 +171,14 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+rX $PRISMA_PATH && \
chmod -R g+rX /app/.cache && \
mkdir -p /tmp/.npm /nonexistent /.npm && \
prisma generate
mkdir -p /tmp/.npm /nonexistent /.npm
# Switch to non-root user for runtime
USER nobody
# Generate Prisma client as nobody user to ensure correct file ownership
RUN prisma generate
# Prisma runtime knobs for offline containers
ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
PRISMA_HIDE_UPDATE_MESSAGE=1 \

1
docker/prod_entrypoint.sh
View file

@ -2,6 +2,7 @@
if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
export LITELLM_ARGS="$@"
export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
exec supervisord -c /etc/supervisord.conf
fi

4
docker/supervisord.conf
View file

@ -1,6 +1,8 @@
[supervisord]
nodaemon=true
loglevel=info
logfile=/tmp/supervisord.log
pidfile=/tmp/supervisord.pid
[group:litellm]
programs=main,health
@ -14,6 +16,7 @@ priority=1
exitcodes=0
stopasgroup=true
killasgroup=true
stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr
stdout_logfile_maxbytes = 0
@ -29,6 +32,7 @@ priority=2
exitcodes=0
stopasgroup=true
killasgroup=true
stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr
stdout_logfile_maxbytes = 0

1
docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md
View file

@ -15,6 +15,7 @@ authors:
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Guide to Claude Opus 4.5 and advanced features in LiteLLM: Tool Search, Programmatic Tool Calling, and Effort Parameter."
tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features]
hide_table_of_contents: false
---

1
docs/my-website/blog/gemini_3/index.md
View file

@ -15,6 +15,7 @@ authors:
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK."
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---

1
docs/my-website/blog/gemini_3_flash/index.md
View file

@ -15,6 +15,7 @@ authors:
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support."
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---

116
docs/my-website/docs/a2a.md
View file

@ -68,7 +68,7 @@ Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./pr
## Invoking your Agents
Use the [A2A Python SDK](https://pypi.org/project/a2a/) to invoke agents through LiteLLM.
Use the [A2A Python SDK](https://pypi.org/project/a2a-sdk) to invoke agents through LiteLLM.
This example shows how to:
1. **List available agents** - Query `/v1/agents` to see which agents your key can access
@ -193,6 +193,120 @@ The logs show:
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
## Forwarding LiteLLM Context Headers
When LiteLLM invokes your A2A agent, it sends special headers that enable:
- **Trace Grouping**: All LLM calls from the same agent execution appear under one trace
- **Agent Spend Tracking**: Costs are attributed to the specific agent
| Header | Purpose |
|--------|---------|
| `X-LiteLLM-Trace-Id` | Links all LLM calls to the same execution flow |
| `X-LiteLLM-Agent-Id` | Attributes spend to the correct agent |
To enable these features, your A2A server must **forward these headers** to any LLM calls it makes back to LiteLLM.
### Implementation Steps
**Step 1: Extract headers from incoming A2A request**
```python def get_litellm_headers(request) -> dict:
"""Extract X-LiteLLM-* headers from incoming A2A request."""
all_headers = request.call_context.state.get('headers', {})
return {
k: v for k, v in all_headers.items()
if k.lower().startswith('x-litellm-')
}
```
**Step 2: Forward headers to your LLM calls**
Pass the extracted headers when making calls back to LiteLLM:
<Tabs>
<TabItem value="openai" label="OpenAI SDK" default>
```python from openai import OpenAI
headers = get_litellm_headers(request)
client = OpenAI(
api_key="sk-your-litellm-key",
base_url="http://localhost:4000",
default_headers=headers, # Forward headers
)
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}]
)
```
</TabItem>
<TabItem value="langchain" label="LangChain">
```python
from langchain_openai import ChatOpenAI
headers = get_litellm_headers(request)
llm = ChatOpenAI(
model="gpt-4o",
openai_api_key="sk-your-litellm-key",
base_url="http://localhost:4000",
default_headers=headers, # Forward headers
)
```
</TabItem>
<TabItem value="litellm" label="LiteLLM SDK">
```python
import litellm
headers = get_litellm_headers(request)
response = litellm.completion(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}],
api_base="http://localhost:4000",
extra_headers=headers, # Forward headers
)
```
</TabItem>
<TabItem value="requests" label="HTTP (requests/httpx)">
```python
import httpx
headers = get_litellm_headers(request)
headers["Authorization"] = "Bearer sk-your-litellm-key"
response = httpx.post(
"http://localhost:4000/v1/chat/completions",
headers=headers,
json={"model": "gpt-4o", "messages": [{"role": "user", "content": "Hello"}]}
)
```
</TabItem>
</Tabs>
### Result
With header forwarding enabled, you'll see:
**Trace Grouping in Langfuse:**
<Image
img={require('../img/a2a_trace_grouping.png')}
style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
/>
**Agent Spend Attribution:**
<Image
img={require('../img/a2a_agent_spend.png')}
style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
/>
## API Reference
### Endpoint

21
docs/my-website/docs/adding_provider/generic_guardrail_api.md
View file

@ -237,6 +237,27 @@ litellm_settings:
language: "en"
```
### Example: Pillar Security
[Pillar Security](https://pillar.security) uses the Generic Guardrail API to provide comprehensive AI security scanning including prompt injection protection, PII/PCI detection, secret detection, and content moderation.
```yaml
guardrails:
- guardrail_name: "pillar-security"
litellm_params:
guardrail: generic_guardrail_api
mode: [pre_call, post_call]
api_base: https://api.pillar.security/api/v1/integrations/litellm
api_key: os.environ/PILLAR_API_KEY
default_on: true
additional_provider_specific_params:
plr_mask: true # Enable automatic masking of sensitive data
plr_evidence: true # Include detection evidence in response
plr_scanners: true # Include scanner details in response
```
See the [Pillar Security documentation](../proxy/guardrails/pillar_security.md) for full configuration options.
## Usage
Users apply your guardrail by name:

0
docs/my-website/docs/anthropic_unified.md → docs/my-website/docs/anthropic_unified/index.md
View file

294
docs/my-website/docs/anthropic_unified/structured_output.md Normal file
View file

@ -0,0 +1,294 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Structured Output /v1/messages
Use LiteLLM to call Anthropic's structured output feature via the `/v1/messages` endpoint.
## Supported Providers
| Provider | Supported | Notes |
|----------|-----------|-------|
| Anthropic | ✅ | Native support |
| Azure AI (Anthropic models) | ✅ | Claude models on Azure AI |
| Bedrock (Converse Anthropic models) | ✅ | Claude models via Bedrock Converse API |
| Bedrock (Invoke Anthropic models) | ✅ | Claude models via Bedrock Invoke API |
## Usage
### LiteLLM Proxy Server
<Tabs>
<TabItem value="anthropic" label="Anthropic">
1. Setup config.yaml
```yaml
model_list:
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-5-20250514
api_key: os.environ/ANTHROPIC_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://localhost:4000/v1/messages \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-H "anthropic-version: 2023-06-01" \
-d '{
"model": "claude-sonnet",
"max_tokens": 1024,
"messages": [
{
"role": "user",
"content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
}
],
"output_format": {
"type": "json_schema",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"email": {"type": "string"},
"plan_interest": {"type": "string"},
"demo_requested": {"type": "boolean"}
},
"required": ["name", "email", "plan_interest", "demo_requested"],
"additionalProperties": false
}
}
}'
```
</TabItem>
<TabItem value="azure_ai" label="Azure AI (Anthropic)">
1. Setup config.yaml
```yaml
model_list:
- model_name: azure-claude-sonnet
litellm_params:
model: azure_ai/claude-sonnet-4-5-20250514
api_key: os.environ/AZURE_AI_API_KEY
api_base: https://your-endpoint.inference.ai.azure.com
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://localhost:4000/v1/messages \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-H "anthropic-version: 2023-06-01" \
-d '{
"model": "azure-claude-sonnet",
"max_tokens": 1024,
"messages": [
{
"role": "user",
"content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
}
],
"output_format": {
"type": "json_schema",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"email": {"type": "string"},
"plan_interest": {"type": "string"},
"demo_requested": {"type": "boolean"}
},
"required": ["name", "email", "plan_interest", "demo_requested"],
"additionalProperties": false
}
}
}'
```
</TabItem>
<TabItem value="bedrock" label="Bedrock (Converse)">
1. Setup config.yaml
```yaml
model_list:
- model_name: bedrock-claude-sonnet
litellm_params:
model: bedrock/global.anthropic.claude-sonnet-4-5-20250929-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://localhost:4000/v1/messages \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-H "anthropic-version: 2023-06-01" \
-d '{
"model": "bedrock-claude-sonnet",
"max_tokens": 1024,
"messages": [
{
"role": "user",
"content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
}
],
"output_format": {
"type": "json_schema",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"email": {"type": "string"},
"plan_interest": {"type": "string"},
"demo_requested": {"type": "boolean"}
},
"required": ["name", "email", "plan_interest", "demo_requested"],
"additionalProperties": false
}
}
}'
```
</TabItem>
<TabItem value="bedrock_invoke" label="Bedrock (Invoke)">
1. Setup config.yaml
```yaml
model_list:
- model_name: bedrock-claude-invoke
litellm_params:
model: bedrock/invoke/global.anthropic.claude-sonnet-4-5-20250929-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://localhost:4000/v1/messages \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-H "anthropic-version: 2023-06-01" \
-d '{
"model": "bedrock-claude-invoke",
"max_tokens": 1024,
"messages": [
{
"role": "user",
"content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
}
],
"output_format": {
"type": "json_schema",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"email": {"type": "string"},
"plan_interest": {"type": "string"},
"demo_requested": {"type": "boolean"}
},
"required": ["name", "email", "plan_interest", "demo_requested"],
"additionalProperties": false
}
}
}'
```
</TabItem>
</Tabs>
## Example Response
```json
{
"id": "msg_01XFDUDYJgAACzvnptvVoYEL",
"type": "message",
"role": "assistant",
"content": [
{
"type": "text",
"text": "{\"name\":\"John Smith\",\"email\":\"john@example.com\",\"plan_interest\":\"Enterprise\",\"demo_requested\":true}"
}
],
"model": "claude-sonnet-4-5-20250514",
"stop_reason": "end_turn",
"stop_sequence": null,
"usage": {
"input_tokens": 75,
"output_tokens": 28
}
}
```
## Request Format
### output_format
The `output_format` parameter specifies the structured output format.
```json
{
"output_format": {
"type": "json_schema",
"schema": {
"type": "object",
"properties": {
"field_name": {"type": "string"},
"another_field": {"type": "integer"}
},
"required": ["field_name", "another_field"],
"additionalProperties": false
}
}
}
```
#### Fields
- **type** (string): Must be `"json_schema"`
- **schema** (object): A JSON Schema object defining the expected output structure
- **type** (string): The root type, typically `"object"`
- **properties** (object): Defines the fields and their types
- **required** (array): List of required field names
- **additionalProperties** (boolean): Set to `false` to enforce strict schema adherence

22
docs/my-website/docs/benchmarks.md
View file

@ -48,6 +48,28 @@ In these tests the baseline latency characteristics are measured against a fake-
- High-percentile latencies drop significantly: P95 630 ms → 150 ms, P99 1,200 ms → 240 ms.
- Setting workers equal to CPU count gives optimal performance.
## `/realtime` API Benchmarks
End-to-end latency benchmarks for the `/realtime` endpoint tested against a fake realtime endpoint.
### Performance Metrics
| Metric | Value |
| --------------- | ---------- |
| Median latency | 59 ms |
| p95 latency | 67 ms |
| p99 latency | 99 ms |
| Average latency | 63 ms |
| RPS | 1,207 |
### Test Setup
| Category | Specification |
|----------|---------------|
| **Load Testing** | Locust: 1,000 concurrent users, 500 ramp-up |
| **System** | 4 vCPUs, 8 GB RAM, 4 workers, 4 instances |
| **Database** | PostgreSQL (Redis unused) |
## Machine Spec used for testing
Each machine deploying LiteLLM had the following specs:

8
docs/my-website/docs/caching/all_caches.md
View file

@ -105,6 +105,14 @@ Then simply initialize:
litellm.cache = Cache(type="redis")
```
:::info
Use `REDIS_*` environment variables as the primary mechanism for configuring all Redis client library parameters. This approach automatically maps environment variables to Redis client kwargs and is the suggested way to toggle Redis settings.
:::
:::warning
If you need to pass non-string Redis parameters (integers, booleans, complex objects), avoid `REDIS_*` environment variables as they may fail during Redis client initialization. Instead, pass them directly as kwargs to the `Cache()` constructor.
:::
</TabItem>
<TabItem value="gcs" label="gcs-cache">

2
docs/my-website/docs/completion/input.md
View file

@ -199,6 +199,8 @@ messages=[{"role": "user", "content": [
- `include_usage` *boolean (optional)* - If set, an additional chunk will be streamed before the data: [DONE] message. The usage field on this chunk shows the token usage statistics for the entire request, and the choices field will always be an empty array. All other chunks will also include a usage field, but with a null value.
- `stop`: *string/ array/ null (optional)* - Up to 4 sequences where the API will stop generating further tokens.
**Note**: OpenAI supports a maximum of 4 stop sequences. If you provide more than 4, LiteLLM will automatically truncate the list to the first 4 elements. To disable this automatic truncation, set `litellm.disable_stop_sequence_limit = True`.
- `max_completion_tokens`: *integer (optional)* - An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens.

88
docs/my-website/docs/completion/json_mode.md
View file

@ -341,4 +341,90 @@ curl http://0.0.0.0:4000/v1/chat/completions \
```
</TabItem>
</Tabs>
</Tabs>
## Gemini - Native JSON Schema Format (Gemini 2.0+)
Gemini 2.0+ models automatically use the native `responseJsonSchema` parameter, which provides better compatibility with standard JSON Schema format.
### Benefits (Gemini 2.0+):
- Standard JSON Schema format (lowercase types like `string`, `object`)
- Supports `additionalProperties: false` for stricter validation
- Better compatibility with Pydantic's `model_json_schema()`
- No `propertyOrdering` required
### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
from pydantic import BaseModel
class UserInfo(BaseModel):
name: str
age: int
response = completion(
model="gemini/gemini-2.0-flash",
messages=[{"role": "user", "content": "Extract: John is 25 years old"}],
response_format={
"type": "json_schema",
"json_schema": {
"name": "user_info",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "integer"}
},
"required": ["name", "age"],
"additionalProperties": False # Supported on Gemini 2.0+
}
}
}
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "gemini-2.0-flash",
"messages": [
{"role": "user", "content": "Extract: John is 25 years old"}
],
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "user_info",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "integer"}
},
"required": ["name", "age"],
"additionalProperties": false
}
}
}
}'
```
</TabItem>
</Tabs>
### Model Behavior
| Model | Format Used | `additionalProperties` Support |
|-------|-------------|-------------------------------|
| Gemini 2.0+ | `responseJsonSchema` (JSON Schema) | ✅ Yes |
| Gemini 1.5 | `responseSchema` (OpenAPI) | ❌ No |
LiteLLM automatically selects the appropriate format based on the model version.

4
docs/my-website/docs/completion/token_usage.md
View file

@ -100,7 +100,7 @@ from litellm import cost_per_token
prompt_tokens = 5
completion_tokens = 10
prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens))
prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)
print(prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar)
```
@ -162,7 +162,7 @@ print(model_cost) # {'gpt-3.5-turbo': {'max_tokens': 4000, 'input_cost_per_token
**Dictionary**
```python
from litellm import register_model
import litellm
litellm.register_model({
"gpt-4": {

101
docs/my-website/docs/contributing.md
View file

@ -1,45 +1,100 @@
# Contributing - UI
Here's how to run the LiteLLM UI locally for making changes:
Thanks for contributing to the LiteLLM UI! This guide will help you set up your local development environment.
## 1. Clone the repo
## 1. Clone the repo
```bash
git clone https://github.com/BerriAI/litellm.git
cd litellm
```
## 2. Start the UI + Proxy
## 2. Start the Proxy
**2.1 Start the proxy on port 4000**
Create a config file (e.g., `config.yaml`):
Tell the proxy where the UI is located
```bash
DATABASE_URL = "postgresql://<user>:<password>@<host>:<port>/<dbname>"
LITELLM_MASTER_KEY = "sk-1234"
STORE_MODEL_IN_DB = "True"
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
general_settings:
master_key: sk-1234
database_url: postgresql://<user>:<password>@<host>:<port>/<dbname>
store_model_in_db: true
```
Start the proxy on port 4000:
```bash
cd litellm/litellm/proxy
python3 proxy_cli.py --config /path/to/config.yaml --port 4000
poetry run litellm --config config.yaml --port 4000
```
**2.2 Start the UI**
The UI comes pre-built in the repo. Access it at `http://localhost:4000/ui`
Set the mode as development (this will assume the proxy is running on localhost:4000)
```bash
npm install # install dependencies
```
## 3. UI Development
There are two options for UI development:
### Option A: Development Mode (Hot Reload)
This runs the UI on port 3000 with hot reload. The proxy runs on port 4000.
```bash
cd litellm/ui/litellm-dashboard
cd ui/litellm-dashboard
npm install
npm run dev
# starts on http://0.0.0.0:3000
```
## 3. Go to local UI
**Login flow:**
1. Go to `http://localhost:3000`
2. You'll be redirected to `http://localhost:4000/ui` for login
3. After logging in, manually navigate back to `http://localhost:3000/`
4. You're now authenticated and can develop with hot reload
:::note
If you experience redirect loops or authentication issues, clear your browser cookies for localhost or use Build Mode instead.
:::
### Option B: Build Mode
This builds the UI and copies it to the proxy. Changes require rebuilding.
1. Make your code changes in `ui/litellm-dashboard/src/`
2. Build the UI
```bash
cd ui/litellm-dashboard
npm install
npm run build
```
After building, copy the output to the proxy:
```bash
http://0.0.0.0:3000
```
cp -r out/* ../../litellm/proxy/_experimental/out/
```
Then restart the proxy and access the UI at `http://localhost:4000/ui`
## 4. Submitting a PR
1. Create a new branch for your changes:
```bash
git checkout -b feat/your-feature-name
```
2. Stage and commit your changes:
```bash
git add .
git commit -m "feat: description of your changes"
```
3. Push to your fork:
```bash
git push origin feat/your-feature-name
```
4. Create a Pull Request on GitHub following the [PR template](https://github.com/BerriAI/litellm/blob/main/.github/pull_request_template.md)

33
docs/my-website/docs/guides/security_settings.md
View file

@ -187,4 +187,37 @@ export AIOHTTP_TRUST_ENV='True'
```
</TabItem>
</Tabs>
## 7. Per-Service SSL Verification
LiteLLM allows you to override SSL verification settings for specific services or provider calls. This is useful when different services (e.g., an internal guardrail vs. a public LLM provider) require different CA certificates.
### Bedrock (SDK)
You can pass `ssl_verify` directly in the `completion` call.
```python
import litellm
response = litellm.completion(
model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
messages=[{"role": "user", "content": "hi"}],
ssl_verify="path/to/bedrock_cert.pem" # Or False to disable
)
```
### AIM Guardrail (Proxy)
You can configure `ssl_verify` per guardrail in your `config.yaml`.
```yaml
guardrails:
- guardrail_name: aim-protected-app
litellm_params:
guardrail: aim
ssl_verify: "/path/to/aim_cert.pem" # Use specific cert for AIM
```
### Priority Logic
LiteLLM resolves `ssl_verify` using the following priority:
1. **Explicit Parameter**: Passed in `completion()` or guardrail config.
2. **Environment Variable**: `SSL_VERIFY` environment variable.
3. **Global Setting**: `litellm.ssl_verify` setting.
4. **System Standard**: `SSL_CERT_FILE` environment variable.

24
docs/my-website/docs/image_generation.md
View file

@ -15,7 +15,7 @@ import TabItem from '@theme/TabItem';
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input prompts (non-streaming only) |
| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, Xinference, Nscale | |
| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, OpenRouter, Xinference, Nscale | |
## Quick Start
@ -238,6 +238,27 @@ print(response)
See Recraft usage with LiteLLM [here](./providers/recraft.md#image-generation)
## OpenRouter Image Generation Models
Use this for image generation models available through OpenRouter (e.g., Google Gemini image generation models)
#### Usage
```python showLineNumbers
from litellm import image_generation
import os
os.environ['OPENROUTER_API_KEY'] = "your-api-key"
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A beautiful sunset over a calm ocean",
size="1024x1024",
quality="high",
)
print(response)
```
## OpenAI Compatible Image Generation Models
Use this for calling `/image_generation` endpoints on OpenAI Compatible Servers, example https://github.com/xorbitsai/inference
@ -301,5 +322,6 @@ print(f"response: {response}")
| Vertex AI | [Vertex AI Image Generation →](./providers/vertex_image) |
| AWS Bedrock | [Bedrock Image Generation →](./providers/bedrock) |
| Recraft | [Recraft Image Generation →](./providers/recraft#image-generation) |
| OpenRouter | [OpenRouter Image Generation →](./providers/openrouter#image-generation) |
| Xinference | [Xinference Image Generation →](./providers/xinference#image-generation) |
| Nscale | [Nscale Image Generation →](./providers/nscale#image-generation) |

5
docs/my-website/docs/mcp.md
View file

@ -21,6 +21,11 @@ LiteLLM Proxy provides an MCP Gateway that allows you to use a fixed endpoint fo
| Supported MCP Transports | • Streamable HTTP<br/>• SSE<br/>• Standard Input/Output (stdio) |
| LiteLLM Permission Management | • By Key<br/>• By Team<br/>• By Organization |
:::caution MCP protocol update
Starting in LiteLLM v1.80.18, the LiteLLM MCP protocol version is `2025-11-25`.<br/>
LiteLLM namespaces multiple MCP servers by prefixing each tool name with its MCP server name, so newly created servers now must use names that comply with SEP-986—noncompliant names cannot be added anymore. Existing servers that still violate SEP-986 only emit warnings today, but future MCP-side rollouts may block those names entirely, so we recommend updating any legacy server names proactively before MCP enforcement makes them unusable.
:::
## Adding your MCP
### Prerequisites

52
docs/my-website/docs/observability/datadog.md
View file

@ -7,6 +7,7 @@ import TabItem from '@theme/TabItem';
LiteLLM Supports logging to the following Datdog Integrations:
- `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
- `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
- `datadog_cost_management` [Datadog Cloud Cost Management](#datadog-cloud-cost-management)
- `ddtrace-run` [Datadog Tracing](#datadog-tracing)
## Datadog Logs
@ -73,7 +74,7 @@ Send logs through a local DataDog agent (useful for containerized environments):
```shell
LITELLM_DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent
LITELLM_DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518)
DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth)
DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (Agent handles auth for Logs. REQUIRED for LLM Observability)
DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source
```
@ -84,6 +85,9 @@ When `LITELLM_DD_AGENT_HOST` is set, logs are sent to the agent instead of direc
**Note:** We use `LITELLM_DD_AGENT_HOST` instead of `DD_AGENT_HOST` to avoid conflicts with `ddtrace` which automatically sets `DD_AGENT_HOST` for APM tracing.
> [!IMPORTANT]
> **Datadog LLM Observability**: `DD_API_KEY` is **REQUIRED** even when using the Datadog Agent (`LITELLM_DD_AGENT_HOST`). The agent acts as a proxy but the API key header is mandatory for the LLM Observability endpoint.
**Step 3**: Start the proxy, make a test request
Start proxy
@ -161,6 +165,50 @@ On the Datadog LLM Observability page, you should see that both input messages a
<Image img={require('../../img/dd_llm_obs.png')} />
## Datadog Cloud Cost Management
| Feature | Details |
|---------|---------|
| **What is logged** | Aggregated LLM Costs (FOCUS format) |
| **Events** | Periodic Uploads of Aggregated Cost Data |
| **Product Link** | [Datadog Cloud Cost Management](https://docs.datadoghq.com/cost_management/) |
We will use the `--config` to set `litellm.callbacks = ["datadog_cost_management"]`. This will periodically upload aggregated LLM cost data to Datadog.
**Step 1**: Create a `config.yaml` file and set `litellm_settings`: `success_callback`
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
litellm_settings:
callbacks: ["datadog_cost_management"]
```
**Step 2**: Set Required env variables
```shell
DD_API_KEY="your-api-key"
DD_APP_KEY="your-app-key" # REQUIRED for Cost Management
DD_SITE="us5.datadoghq.com"
```
**Step 3**: Start the proxy
```shell
litellm --config config.yaml
```
**How it works**
* LiteLLM aggregates costs in-memory by Provider, Model, Date, and Tags.
* Requires `DD_APP_KEY` for the Custom Costs API.
* Costs are uploaded periodically (flushed).
### Datadog Tracing
Use `ddtrace-run` to enable [Datadog Tracing](https://ddtrace.readthedocs.io/en/stable/installation_quickstart.html) on litellm proxy
@ -203,5 +251,5 @@ LiteLLM supports customizing the following Datadog environment variables
| `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No |
\* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required
\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required
\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required for **Datadog Logs**. (**Note: `DD_API_KEY` IS REQUIRED for Datadog LLM Observability**)

4
docs/my-website/docs/observability/logfire_integration.md
View file

@ -40,6 +40,10 @@ import os
# from https://logfire.pydantic.dev/
os.environ["LOGFIRE_TOKEN"] = ""
# Optionally customize the base url
# from https://logfire.pydantic.dev/
os.environ["LOGFIRE_BASE_URL"] = ""
# LLM API Keys
os.environ['OPENAI_API_KEY']=""

6
docs/my-website/docs/observability/opentelemetry_integration.md
View file

@ -63,6 +63,8 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc
OTEL_EXPORTER_OTLP_HEADERS="api-key=key,other-config-value=value"
```
> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`).
</TabItem>
<TabItem value="laminar" label="Log to Laminar">
@ -73,6 +75,8 @@ OTEL_ENDPOINT="https://api.lmnr.ai:8443"
OTEL_HEADERS="authorization=Bearer <project-api-key>"
```
> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`).
</TabItem>
</Tabs>
@ -128,4 +132,4 @@ If you don't see traces landing on your integration, set `OTEL_DEBUG="True"` in
export OTEL_DEBUG="True"
```
This will emit any logging issues to the console.
This will emit any logging issues to the console.

2
docs/my-website/docs/observability/phoenix_integration.md
View file

@ -73,6 +73,8 @@ environment_variables:
PHOENIX_COLLECTOR_HTTP_ENDPOINT: "https://app.phoenix.arize.com/s/<space-name>/v1/traces" # OPTIONAL - For setting the HTTP endpoint
```
> Note: If you set the gRPC endpoint, install `grpcio` via `pip install "litellm[grpc]"` (or `grpcio`).
2. Start the proxy
```bash

4
docs/my-website/docs/observability/signoz.md
View file

@ -99,6 +99,8 @@ OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai \
opentelemetry-instrument <your_run_command>
```
> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`).
> 📌 Note: We're using `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai` in the run command to disable the OpenAI instrumentor for tracing. This avoids conflicts with LiteLLM's native telemetry/instrumentation, ensuring that telemetry is captured exclusively through LiteLLM's built-in instrumentation.
- **`<service_name>`** is the name of your service
@ -362,6 +364,8 @@ export OTEL_METRICS_EXPORTER="otlp"
export OTEL_LOGS_EXPORTER="otlp"
```
> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`).
- Set the `<region>` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint)
- Replace `<your_ingestion_key>` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)

25
docs/my-website/docs/pass_through/openai_passthrough.md
View file

@ -1,6 +1,6 @@
# OpenAI Passthrough
Pass-through endpoints for `/openai`
Pass-through endpoints for direct OpenAI API access
## Overview
@ -10,12 +10,27 @@ Pass-through endpoints for `/openai`
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | Fully supported |
### When to use this?
## Available Endpoints
### `/openai_passthrough` - Recommended
Dedicated passthrough endpoint that guarantees direct routing to OpenAI without conflicts.
**Use this for:**
- OpenAI Responses API (`/v1/responses`)
- Any endpoint where you need guaranteed passthrough
- When `/openai` routes are conflicting with LiteLLM's native implementations
### `/openai` - Legacy
Standard passthrough endpoint that may conflict with LiteLLM's native implementations.
**Note:** Some endpoints like `/openai/v1/responses` will be routed to LiteLLM's native implementation instead of OpenAI.
## When to use this?
- For 90% of your use cases, you should use the [native LiteLLM OpenAI Integration](https://docs.litellm.ai/docs/providers/openai) (`/chat/completions`, `/embeddings`, `/completions`, `/images`, `/batches`, etc.)
- Use this passthrough to call less popular or newer OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`
- Use `/openai_passthrough` to call less popular or newer OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`, `/responses`
Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai`
Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai_passthrough`
## Usage Examples
@ -34,7 +49,7 @@ Make sure you do the following:
import openai
client = openai.OpenAI(
base_url="http://0.0.0.0:4000/openai", # <your-proxy-url>/openai
base_url="http://0.0.0.0:4000/openai_passthrough", # <your-proxy-url>/openai_passthrough
api_key="sk-anything" # <your-proxy-api-key>
)
```

51
docs/my-website/docs/pass_through/vertex_ai.md
View file

@ -45,7 +45,7 @@ model_list:
litellm_params:
model: vertex_ai/gemini-1.0-pro
vertex_project: adroit-crow-413218
vertex_region: us-central1
vertex_location: us-central1
vertex_credentials: /path/to/credentials.json
use_in_pass_through: true # 👈 KEY CHANGE
```
@ -57,9 +57,9 @@ model_list:
<TabItem value="yaml" label="Set in config.yaml">
```yaml
default_vertex_config:
default_vertex_config:
vertex_project: adroit-crow-413218
vertex_region: us-central1
vertex_location: us-central1
vertex_credentials: /path/to/credentials.json
```
</TabItem>
@ -461,3 +461,48 @@ generateContent();
</TabItem>
</Tabs>
### Using Anthropic Beta Features on Vertex AI
When using Anthropic models via Vertex AI passthrough (e.g., Claude on Vertex), you can enable Anthropic beta features like extended context windows.
The `anthropic-beta` header is automatically forwarded to Vertex AI when calling Anthropic models.
```bash
curl http://localhost:4000/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-east5/publishers/anthropic/models/claude-3-5-sonnet:rawPredict \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-H "anthropic-beta: context-1m-2025-08-07" \
-d '{
"anthropic_version": "vertex-2023-10-16",
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 500
}'
```
### Forwarding Custom Headers with `x-pass-` Prefix
You can forward any custom header to the provider by prefixing it with `x-pass-`. The prefix is stripped before the header is sent to the provider.
For example:
- `x-pass-anthropic-beta: value` becomes `anthropic-beta: value`
- `x-pass-custom-header: value` becomes `custom-header: value`
This is useful when you need to send provider-specific headers that aren't in the default allowlist.
```bash
curl http://localhost:4000/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-east5/publishers/anthropic/models/claude-3-5-sonnet:rawPredict \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-H "x-pass-anthropic-beta: context-1m-2025-08-07" \
-H "x-pass-custom-feature: enabled" \
-d '{
"anthropic_version": "vertex-2023-10-16",
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 500
}'
```
:::info
The `x-pass-` prefix works for all LLM pass-through endpoints, not just Vertex AI.
:::

450
docs/my-website/docs/providers/anthropic_tool_search.md
View file

@ -1,43 +1,46 @@
# Anthropic Tool Search
# Tool Search
Tool search enables Claude to dynamically discover and load tools on-demand from large tool catalogs (10,000+ tools). Instead of loading all tool definitions into the context window upfront, Claude searches your tool catalog and loads only the tools it needs.
## Supported Providers
| Provider | Chat Completions API | Messages API |
|----------|---------------------|--------------|
| **Anthropic API** | ✅ | ✅ |
| **Azure Anthropic** (Microsoft Foundry) | ✅ | ✅ |
| **Google Cloud Vertex AI** | ✅ | ✅ |
| **Amazon Bedrock** | ✅ (Invoke API only, Opus 4.5 only) | ✅ (Invoke API only, Opus 4.5 only) |
## Benefits
- **Context efficiency**: Avoid consuming massive portions of your context window with tool definitions
- **Better tool selection**: Claude's tool selection accuracy degrades with more than 30-50 tools. Tool search maintains accuracy even with thousands of tools
- **On-demand loading**: Tools are only loaded when Claude needs them
## Supported Models
Tool search is available on:
- Claude Opus 4.5
- Claude Sonnet 4.5
## Supported Platforms
- Anthropic API (direct)
- Azure Anthropic (Microsoft Foundry)
- Google Cloud Vertex AI
- Amazon Bedrock (invoke API only, not converse API)
## Tool Search Variants
LiteLLM supports both tool search variants:
### 1. Regex Tool Search (`tool_search_tool_regex_20251119`)
Claude constructs regex patterns to search for tools.
Claude constructs regex patterns to search for tools. Best for exact pattern matching (faster).
### 2. BM25 Tool Search (`tool_search_tool_bm25_20251119`)
Claude uses natural language queries to search for tools using the BM25 algorithm.
Claude uses natural language queries to search for tools using the BM25 algorithm. Best for natural language semantic search.
## Quick Start
**Note**: BM25 variant is not supported on Bedrock.
### Basic Example with Regex Tool Search
---
```python
## Chat Completions API
### SDK Usage
#### Basic Example with Regex Tool Search
```python showLineNumbers title="Basic Tool Search Example"
import litellm
response = litellm.completion(
@ -70,26 +73,6 @@ response = litellm.completion(
}
},
"defer_loading": True # Mark for deferred loading
},
# Another deferred tool
{
"type": "function",
"function": {
"name": "search_files",
"description": "Search through files in the workspace",
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string"},
"file_types": {
"type": "array",
"items": {"type": "string"}
}
},
"required": ["query"]
}
},
"defer_loading": True
}
]
)
@ -97,9 +80,9 @@ response = litellm.completion(
print(response.choices[0].message.content)
```
### BM25 Tool Search Example
#### BM25 Tool Search Example
```python
```python showLineNumbers title="BM25 Tool Search"
import litellm
response = litellm.completion(
@ -134,9 +117,9 @@ response = litellm.completion(
)
```
## Using with Azure Anthropic
#### Azure Anthropic Example
```python
```python showLineNumbers title="Azure Anthropic Tool Search"
import litellm
response = litellm.completion(
@ -170,9 +153,9 @@ response = litellm.completion(
)
```
## Using with Vertex AI
#### Vertex AI Example
```python
```python showLineNumbers title="Vertex AI Tool Search"
import litellm
response = litellm.completion(
@ -192,11 +175,9 @@ response = litellm.completion(
)
```
## Streaming Support
#### Streaming Support
Tool search works with streaming:
```python
```python showLineNumbers title="Streaming with Tool Search"
import litellm
response = litellm.completion(
@ -233,13 +214,13 @@ for chunk in response:
print(chunk.choices[0].delta.content, end="")
```
## LiteLLM Proxy
### AI Gateway Usage
Tool search works automatically through the LiteLLM proxy:
Tool search works automatically through the LiteLLM proxy.
### Proxy Config
#### Proxy Configuration
```yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: claude-sonnet
litellm_params:
@ -247,18 +228,19 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
```
### Client Request
#### Client Request
```python
import openai
```python showLineNumbers title="Client Request via Proxy"
from anthropic import Anthropic
client = openai.OpenAI(
client = Anthropic(
api_key="your-litellm-proxy-key",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
response = client.messages.create(
model="claude-sonnet",
max_tokens=1024,
messages=[
{"role": "user", "content": "What's the weather?"}
],
@ -268,17 +250,14 @@ response = client.chat.completions.create(
"name": "tool_search_tool_regex"
},
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather information",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
"name": "get_weather",
"description": "Get weather information",
"input_schema": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
},
"defer_loading": True
}
@ -286,127 +265,278 @@ response = client.chat.completions.create(
)
```
## Important Notes
---
### Beta Header
## Messages API
LiteLLM automatically detects tool search tools and adds the appropriate beta header based on your provider:
The Messages API provides native Anthropic-style tool search support via the `litellm.anthropic.messages` interface.
- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20`
- **Google Cloud Vertex AI**: `tool-search-tool-2025-10-19`
- **Amazon Bedrock** (Invoke API, Opus 4.5 only): `tool-search-tool-2025-10-19`
### SDK Usage
You don't need to manually specify beta headers—LiteLLM handles this automatically.
#### Basic Example
### Deferred Loading
```python showLineNumbers title="Messages API - Basic Tool Search"
import litellm
- Tools with `defer_loading: true` are only loaded when Claude discovers them via search
- At least one tool must be non-deferred (the tool search tool itself)
- Keep your 3-5 most frequently used tools as non-deferred for optimal performance
### Tool Descriptions
Write clear, descriptive tool names and descriptions that match how users describe tasks. The search algorithm uses:
- Tool names
- Tool descriptions
- Argument names
- Argument descriptions
### Usage Tracking
Tool search requests are tracked in the usage object:
```python
response = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[{"role": "user", "content": "Search for tools"}],
tools=[...]
response = await litellm.anthropic.messages.acreate(
model="anthropic/claude-sonnet-4-20250514",
messages=[
{
"role": "user",
"content": "What's the weather in San Francisco?"
}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"name": "get_weather",
"description": "Get the current weather for a location",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA"
}
},
"required": ["location"]
},
"defer_loading": True
}
],
max_tokens=1024,
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
)
# Check tool search usage
if response.usage.server_tool_use:
print(f"Tool search requests: {response.usage.server_tool_use.tool_search_requests}")
print(response)
```
## Error Handling
#### Azure Anthropic Messages Example
### All Tools Deferred
```python showLineNumbers title="Azure Anthropic Messages API"
import litellm
```python
# ❌ This will fail - at least one tool must be non-deferred
tools = [
{
"type": "function",
"function": {...},
"defer_loading": True
}
]
# ✅ Correct - tool search tool is non-deferred
tools = [
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"type": "function",
"function": {...},
"defer_loading": True
}
]
response = await litellm.anthropic.messages.acreate(
model="azure_anthropic/claude-sonnet-4-20250514",
messages=[
{
"role": "user",
"content": "What's the stock price of Apple?"
}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"name": "get_stock_price",
"description": "Get the current stock price for a ticker symbol",
"input_schema": {
"type": "object",
"properties": {
"ticker": {
"type": "string",
"description": "The stock ticker symbol, e.g. AAPL"
}
},
"required": ["ticker"]
},
"defer_loading": True
}
],
max_tokens=1024,
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
)
```
### Missing Tool Definition
#### Vertex AI Messages Example
If Claude references a tool that isn't in your deferred tools list, you'll get an error. Make sure all tools that might be discovered are included in the tools parameter with `defer_loading: true`.
```python showLineNumbers title="Vertex AI Messages API"
import litellm
## Best Practices
response = await litellm.anthropic.messages.acreate(
model="vertex_ai/claude-sonnet-4@20250514",
messages=[
{
"role": "user",
"content": "Search the web for information about AI"
}
],
tools=[
{
"type": "tool_search_tool_bm25_20251119",
"name": "tool_search_tool_bm25"
},
{
"name": "search_web",
"description": "Search the web for information",
"input_schema": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "The search query"
}
},
"required": ["query"]
},
"defer_loading": True
}
],
max_tokens=1024,
extra_headers={"anthropic-beta": "tool-search-tool-2025-10-19"}
)
```
1. **Keep frequently used tools non-deferred**: Your 3-5 most common tools should not have `defer_loading: true`
#### Bedrock Messages Example
2. **Use semantic descriptions**: Tool descriptions should use natural language that matches user queries
```python showLineNumbers title="Bedrock Messages API (Invoke)"
import litellm
3. **Choose the right variant**:
- Use **regex** for exact pattern matching (faster)
- Use **BM25** for natural language semantic search
response = await litellm.anthropic.messages.acreate(
model="bedrock/invoke/anthropic.claude-opus-4-20250514-v1:0",
messages=[
{
"role": "user",
"content": "What's the weather?"
}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"name": "get_weather",
"description": "Get weather information",
"input_schema": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
},
"defer_loading": True
}
],
max_tokens=1024,
extra_headers={"anthropic-beta": "tool-search-tool-2025-10-19"}
)
```
4. **Monitor usage**: Track `tool_search_requests` in the usage object to understand search patterns
#### Streaming Support
5. **Optimize tool catalog**: Remove unused tools and consolidate similar functionality
```python showLineNumbers title="Messages API - Streaming"
import litellm
import json
## When to Use Tool Search
response = await litellm.anthropic.messages.acreate(
model="anthropic/claude-sonnet-4-20250514",
messages=[
{
"role": "user",
"content": "What's the weather in Tokyo?"
}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"name": "get_weather",
"description": "Get weather information",
"input_schema": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
},
"defer_loading": True
}
],
max_tokens=1024,
stream=True,
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
)
**Good use cases:**
- 10+ tools available in your system
- Tool definitions consuming >10K tokens
- Experiencing tool selection accuracy issues
- Building systems with multiple tool categories
- Tool library growing over time
async for chunk in response:
if isinstance(chunk, bytes):
chunk_str = chunk.decode("utf-8")
for line in chunk_str.split("\n"):
if line.startswith("data: "):
try:
json_data = json.loads(line[6:])
print(json_data)
except json.JSONDecodeError:
pass
```
**When traditional tool calling is better:**
- Less than 10 tools total
- All tools are frequently used
- Very small tool definitions (\<100 tokens total)
### AI Gateway Usage
## Limitations
Configure the proxy to use Messages API endpoints.
- Not compatible with tool use examples
- Requires Claude Opus 4.5 or Sonnet 4.5
- On Bedrock, only available via invoke API (not converse API)
- On Bedrock, only supported for Claude Opus 4.5 (not Sonnet 4.5)
- BM25 variant (`tool_search_tool_bm25_20251119`) is not supported on Bedrock
- Maximum 10,000 tools in catalog
- Returns 3-5 most relevant tools per search
#### Proxy Configuration
### Bedrock-Specific Notes
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: claude-sonnet-messages
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: os.environ/ANTHROPIC_API_KEY
```
When using Bedrock's Invoke API:
- The regex variant (`tool_search_tool_regex_20251119`) is automatically normalized to `tool_search_tool_regex`
- The BM25 variant (`tool_search_tool_bm25_20251119`) is automatically filtered out as it's not supported
- Tool search is only available for Claude Opus 4.5 models
#### Client Request
```python showLineNumbers title="Client Request via Proxy (Messages API)"
from anthropic import Anthropic
client = Anthropic(
api_key="your-litellm-proxy-key",
base_url="http://0.0.0.0:4000"
)
response = client.messages.create(
model="claude-sonnet-messages",
max_tokens=1024,
messages=[
{
"role": "user",
"content": "What's the weather?"
}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"name": "get_weather",
"description": "Get weather information",
"input_schema": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
},
"defer_loading": True
}
],
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
)
print(response)
```
---
## Additional Resources
- [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search)
- [LiteLLM Tool Calling Guide](https://docs.litellm.ai/docs/completion/function_call)

163
docs/my-website/docs/providers/azure_ai/azure_model_router.md
View file

@ -5,19 +5,38 @@ Azure Model Router is a feature in Azure AI Foundry that automatically routes yo
## Key Features
- **Automatic Model Selection**: Azure Model Router dynamically selects the best model for your request
- **Cost Tracking**: LiteLLM automatically tracks costs based on the actual model used (e.g., `gpt-4.1-nano`), not the router endpoint
- **Cost Tracking**: LiteLLM automatically tracks costs based on the actual model used (e.g., `gpt-4.1-nano`), plus the Model Router infrastructure fee
- **Streaming Support**: Full support for streaming responses with accurate cost calculation
- **Simple Configuration**: Easy to set up via UI or config file
## Model Naming Pattern
Use the pattern: `azure_ai/model_router/<deployment-name>`
**Components:**
- `azure_ai` - The provider identifier
- `model_router` - Indicates this is a Model Router deployment
- `<deployment-name>` - Your actual deployment name from Azure AI Foundry (e.g., `azure-model-router`)
**Example:** `azure_ai/model_router/azure-model-router`
**How it works:**
- LiteLLM automatically strips the `model_router/` prefix when sending requests to Azure
- Only your deployment name (e.g., `azure-model-router`) is sent to the Azure API
- The full path is preserved in responses and logs for proper cost tracking
## LiteLLM Python SDK
### Basic Usage
Use the pattern `azure_ai/model_router/<deployment-name>` where `<deployment-name>` is your Azure deployment name:
```python
import litellm
import os
response = litellm.completion(
model="azure_ai/azure-model-router",
model="azure_ai/model_router/azure-model-router", # Use your deployment name
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
@ -26,6 +45,13 @@ response = litellm.completion(
print(response)
```
**Pattern Explanation:**
- `azure_ai` - The provider
- `model_router` - Indicates this is a model router deployment
- `azure-model-router` - Your actual deployment name from Azure AI Foundry
LiteLLM will automatically strip the `model_router/` prefix when sending the request to Azure, so only `azure-model-router` is sent to the API.
### Streaming with Usage Tracking
```python
@ -33,7 +59,7 @@ import litellm
import os
response = await litellm.acompletion(
model="azure_ai/azure-model-router",
model="azure_ai/model_router/azure-model-router", # Use your deployment name
messages=[{"role": "user", "content": "hi"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
@ -51,13 +77,15 @@ async for chunk in response:
```yaml
model_list:
- model_name: azure-model-router
- model_name: azure-model-router # Public name for your users
litellm_params:
model: azure_ai/azure-model-router
model: azure_ai/model_router/azure-model-router # Use your deployment name
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/v1/
api_key: os.environ/AZURE_MODEL_ROUTER_API_KEY
```
**Note:** Replace `azure-model-router` in the model path with your actual deployment name from Azure AI Foundry.
### Start Proxy
```bash
@ -80,49 +108,42 @@ curl -X POST http://localhost:4000/chat/completions \
This walkthrough shows how to add an Azure Model Router endpoint to LiteLLM using the Admin Dashboard.
### Select Provider
### Quick Start
1. Navigate to the **Models** page in the LiteLLM UI
2. Select **"Azure AI Foundry (Studio)"** as the provider
3. Enter your deployment name (e.g., `azure-model-router`)
4. LiteLLM will automatically format it as `azure_ai/model_router/azure-model-router`
5. Add your API base URL and API key
6. Test and save
### Detailed Walkthrough
#### Step 1: Select Provider
Navigate to the Models page and select "Azure AI Foundry (Studio)" as the provider.
#### Navigate to Models Page
##### Navigate to Models Page
![Navigate to Models](./img/azure_model_router_01.jpeg)
#### Click Provider Dropdown
##### Click Provider Dropdown
![Click Provider](./img/azure_model_router_02.jpeg)
#### Choose Azure AI Foundry
##### Choose Azure AI Foundry
![Select Azure AI Foundry](./img/azure_model_router_03.jpeg)
### Configure Model Name
#### Step 2: Enter Deployment Name
Set up the model name by entering `azure_ai/` followed by your model router deployment name from Azure.
**New Simplified Method:** Just enter your deployment name directly in the text field. If your deployment name contains "model-router" or "model_router", LiteLLM will automatically format it as `azure_ai/model_router/<deployment-name>`.
#### Click Model Name Field
**Example:**
- Enter: `azure-model-router`
- LiteLLM creates: `azure_ai/model_router/azure-model-router`
![Click Model Field](./img/azure_model_router_04.jpeg)
#### Select Custom Model Name
![Select Custom Model](./img/azure_model_router_05.jpeg)
#### Enter LiteLLM Model Name
![LiteLLM Model Name](./img/azure_model_router_06.jpeg)
#### Click Custom Model Name Field
![Enter Custom Name Field](./img/azure_model_router_07.jpeg)
#### Type Model Prefix
Type `azure_ai/` as the prefix.
![Type azure_ai prefix](./img/azure_model_router_08.jpeg)
#### Copy Model Name from Azure Portal
##### Copy Deployment Name from Azure Portal
Switch to Azure AI Foundry and copy your model router deployment name.
@ -130,73 +151,79 @@ Switch to Azure AI Foundry and copy your model router deployment name.
![Copy Model Name](./img/azure_model_router_10.jpeg)
#### Paste Model Name
##### Enter Deployment Name in LiteLLM
Paste to get `azure_ai/azure-model-router`.
Paste your deployment name (e.g., `azure-model-router`) directly into the text field.
![Paste Model Name](./img/azure_model_router_11.jpeg)
![Enter Deployment Name](./img/azure_model_router_04.jpeg)
### Configure API Base and Key
**What happens behind the scenes:**
- You enter: `azure-model-router`
- LiteLLM automatically detects this is a model router deployment
- The full model path becomes: `azure_ai/model_router/azure-model-router`
- When making API calls, only `azure-model-router` is sent to Azure
#### Step 3: Configure API Base and Key
Copy the endpoint URL and API key from Azure portal.
#### Copy API Base URL from Azure
##### Copy API Base URL from Azure
![Copy API Base](./img/azure_model_router_12.jpeg)
#### Enter API Base in LiteLLM
##### Enter API Base in LiteLLM
![Click API Base Field](./img/azure_model_router_13.jpeg)
![Paste API Base](./img/azure_model_router_14.jpeg)
#### Copy API Key from Azure
##### Copy API Key from Azure
![Copy API Key](./img/azure_model_router_15.jpeg)
#### Enter API Key in LiteLLM
##### Enter API Key in LiteLLM
![Enter API Key](./img/azure_model_router_16.jpeg)
### Test and Add Model
#### Step 4: Test and Add Model
Verify your configuration works and save the model.
#### Test Connection
##### Test Connection
![Test Connection](./img/azure_model_router_17.jpeg)
#### Close Test Dialog
##### Close Test Dialog
![Close Dialog](./img/azure_model_router_18.jpeg)
#### Add Model
##### Add Model
![Add Model](./img/azure_model_router_19.jpeg)
### Verify in Playground
#### Step 5: Verify in Playground
Test your model and verify cost tracking is working.
#### Open Playground
##### Open Playground
![Go to Playground](./img/azure_model_router_20.jpeg)
#### Select Model
##### Select Model
![Select Model](./img/azure_model_router_21.jpeg)
#### Send Test Message
##### Send Test Message
![Send Message](./img/azure_model_router_22.jpeg)
#### View Logs
##### View Logs
![View Logs](./img/azure_model_router_23.jpeg)
#### Verify Cost Tracking
##### Verify Cost Tracking
Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`).
Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`), plus a flat infrastructure cost of $0.14 per million input tokens for using the Model Router.
![Verify Cost](./img/azure_model_router_24.jpeg)
@ -205,28 +232,50 @@ Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`).
LiteLLM automatically handles cost tracking for Azure Model Router by:
1. **Detecting the actual model**: When Azure Model Router routes your request to a specific model (e.g., `gpt-4.1-nano-2025-04-14`), LiteLLM extracts this from the response
2. **Calculating accurate costs**: Costs are calculated based on the actual model used, not the router endpoint name
2. **Calculating accurate costs**: Costs are calculated based on:
- The actual model used (e.g., `gpt-4.1-nano` token costs)
- Plus a flat infrastructure cost of **$0.14 per million input tokens** for using the Model Router
3. **Streaming support**: Cost tracking works correctly for both streaming and non-streaming requests
### Cost Breakdown
When you use Azure Model Router, the total cost includes:
- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-4.1-nano`)
- **Router Flat Cost**: $0.14 per million input tokens (Azure AI Foundry infrastructure fee)
### Example Response with Cost
```python
import litellm
response = litellm.completion(
model="azure_ai/azure-model-router",
model="azure_ai/model_router/azure-model-router",
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key="your-api-key",
)
# The response will show the actual model used
print(f"Model used: {response.model}") # e.g., "gpt-4.1-nano-2025-04-14"
print(f"Model used: {response.model}") # e.g., "azure_ai/gpt-4.1-nano-2025-04-14"
# Get cost
# Get cost (includes both model cost and router flat cost)
from litellm import completion_cost
cost = completion_cost(completion_response=response)
print(f"Cost: ${cost}")
print(f"Total cost: ${cost}")
# Access detailed cost breakdown
if hasattr(response, '_hidden_params') and 'response_cost' in response._hidden_params:
print(f"Response cost: ${response._hidden_params['response_cost']}")
```
### Viewing Cost Breakdown in UI
When viewing logs in the LiteLLM UI, you'll see:
- **Model Cost**: The cost for the actual model used
- **Azure Model Router Flat Cost**: The $0.14/M input tokens infrastructure fee
- **Total Cost**: Sum of both costs
This breakdown helps you understand exactly what you're paying for when using the Model Router.

84
docs/my-website/docs/providers/chatgpt.md Normal file
View file

@ -0,0 +1,84 @@
# ChatGPT Subscription
Use ChatGPT Pro/Max subscription models through LiteLLM with OAuth device flow authentication.
| Property | Details |
|-------|-------|
| Description | ChatGPT subscription access (Codex + GPT-5.2 family) via ChatGPT backend API |
| Provider Route on LiteLLM | `chatgpt/` |
| Supported Endpoints | `/responses`, `/chat/completions` (bridged to Responses for supported models) |
| API Reference | https://chatgpt.com |
ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.2`).
Notes:
- The ChatGPT subscription backend rejects token limit fields (`max_tokens`, `max_output_tokens`, `max_completion_tokens`) and `metadata`. LiteLLM strips these fields for this provider.
- `/v1/chat/completions` honors `stream`. When `stream` is false (default), LiteLLM aggregates the Responses stream into a single JSON response.
## Authentication
ChatGPT subscription access uses an OAuth device code flow:
1. LiteLLM prints a device code and verification URL
2. Open the URL, sign in, and enter the code
3. Tokens are stored locally for reuse
## Usage - LiteLLM Python SDK
### Responses (recommended for Codex models)
```python showLineNumbers title="ChatGPT Responses"
import litellm
response = litellm.responses(
model="chatgpt/gpt-5.2-codex",
input="Write a Python hello world"
)
print(response)
```
### Chat Completions (bridged to Responses)
```python showLineNumbers title="ChatGPT Chat Completions"
import litellm
response = litellm.completion(
model="chatgpt/gpt-5.2",
messages=[{"role": "user", "content": "Write a Python hello world"}]
)
print(response)
```
## Usage - LiteLLM Proxy
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: chatgpt/gpt-5.2
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.2
- model_name: chatgpt/gpt-5.2-codex
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.2-codex
```
```bash showLineNumbers title="Start LiteLLM Proxy"
litellm --config config.yaml
```
## Configuration
### Environment Variables
- `CHATGPT_TOKEN_DIR`: Custom token storage directory
- `CHATGPT_AUTH_FILE`: Auth file name (default: `auth.json`)
- `CHATGPT_API_BASE`: Override API base (default: `https://chatgpt.com/backend-api/codex`)
- `OPENAI_CHATGPT_API_BASE`: Alias for `CHATGPT_API_BASE`
- `CHATGPT_ORIGINATOR`: Override the `originator` header value
- `CHATGPT_USER_AGENT`: Override the `User-Agent` header value
- `CHATGPT_USER_AGENT_SUFFIX`: Optional suffix appended to the `User-Agent` header

258
docs/my-website/docs/providers/gemini.md
View file

@ -15,6 +15,17 @@ import TabItem from '@theme/TabItem';
<br />
:::tip Gemini API vs Vertex AI
| Model Format | Provider | Auth Required |
|-------------|----------|---------------|
| `gemini/gemini-2.0-flash` | Gemini API | `GEMINI_API_KEY` (simple API key) |
| `vertex_ai/gemini-2.0-flash` | Vertex AI | GCP credentials + project |
| `gemini-2.0-flash` (no prefix) | Vertex AI | GCP credentials + project |
**If you just want to use an API key** (like OpenAI), use the `gemini/` prefix.
Models without a prefix default to Vertex AI which requires full GCP authentication.
:::
## API Keys
@ -1547,16 +1558,21 @@ LiteLLM Supports the following image types passed in `url`
- Images with direct links - https://storage.googleapis.com/github-repo/img/gemini/intro/landmark3.jpg
- Image in local storage - ./localimage.jpeg
## Image Resolution Control (Gemini 3+)
## Media Resolution Control (Images & Videos)
For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images in your request.
For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types.
**Supported `detail` values:**
- `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos)
- `"medium"` - Maps to `media_resolution: "medium"`
- `"high"` - Maps to `media_resolution: "high"` (1120 tokens for images)
- `"ultra_high"` - Maps to `media_resolution: "ultra_high"`
- `"auto"` or `None` - Model decides optimal resolution (no `media_resolution` set)
**Usage Example:**
**Usage Examples:**
<Tabs>
<TabItem value="images" label="Images">
```python
from litellm import completion
@ -1593,10 +1609,193 @@ response = completion(
)
```
</TabItem>
<TabItem value="videos" label="Videos with Files">
```python
from litellm import completion
messages = [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Analyze this video"
},
{
"type": "file",
"file": {
"file_id": "gs://my-bucket/video.mp4",
"format": "video/mp4",
"detail": "high" # High resolution for detailed video analysis
}
}
]
}
]
response = completion(
model="gemini/gemini-3-pro-preview",
messages=messages,
)
```
</TabItem>
</Tabs>
:::info
**Per-Part Resolution:** Each image in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature is only available for Gemini 3+ models.
**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types, and is only available for Gemini 3+ models.
:::
## Video Metadata Control
For Gemini 3+ models, LiteLLM supports fine-grained video processing control through the `video_metadata` field. This allows you to specify frame extraction rates and time ranges for video analysis.
**Supported `video_metadata` parameters:**
| Parameter | Type | Description | Example |
|-----------|------|-------------|---------|
| `fps` | Number | Frame extraction rate (frames per second) | `5` |
| `start_offset` | String | Start time for video clip processing | `"10s"` |
| `end_offset` | String | End time for video clip processing | `"60s"` |
:::note
**Field Name Conversion:** LiteLLM automatically converts snake_case field names to camelCase for the Gemini API:
- `start_offset` → `startOffset`
- `end_offset` → `endOffset`
- `fps` remains unchanged
:::
:::warning
- **Gemini 3+ Only:** This feature is only available for Gemini 3.0 and newer models
- **Video Files Recommended:** While `video_metadata` is designed for video files, error handling for other media types is delegated to the Vertex AI API
- **File Formats Supported:** Works with `gs://`, `https://`, and base64-encoded video files
:::
**Usage Examples:**
<Tabs>
<TabItem value="basic" label="Basic Video Metadata">
```python
from litellm import completion
response = completion(
model="gemini/gemini-3-pro-preview",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this video clip"},
{
"type": "file",
"file": {
"file_id": "gs://my-bucket/video.mp4",
"format": "video/mp4",
"video_metadata": {
"fps": 5, # Extract 5 frames per second
"start_offset": "10s", # Start from 10 seconds
"end_offset": "60s" # End at 60 seconds
}
}
}
]
}
]
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="combined" label="Combined with Detail">
```python
from litellm import completion
response = completion(
model="gemini/gemini-3-pro-preview",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "Provide detailed analysis of this video segment"},
{
"type": "file",
"file": {
"file_id": "https://example.com/presentation.mp4",
"format": "video/mp4",
"detail": "high", # High resolution for detailed analysis
"video_metadata": {
"fps": 10, # Extract 10 frames per second
"start_offset": "30s", # Start from 30 seconds
"end_offset": "90s" # End at 90 seconds
}
}
}
]
}
]
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: gemini-3-pro
litellm_params:
model: gemini/gemini-3-pro-preview
api_key: os.environ/GEMINI_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Make request
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-d '{
"model": "gemini-3-pro",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this video clip"},
{
"type": "file",
"file": {
"file_id": "gs://my-bucket/video.mp4",
"format": "video/mp4",
"detail": "high",
"video_metadata": {
"fps": 5,
"start_offset": "10s",
"end_offset": "60s"
}
}
}
]
}
]
}'
```
</TabItem>
</Tabs>
## Sample Usage
```python
import os
@ -1641,6 +1840,57 @@ content = response.get('choices', [{}])[0].get('message', {}).get('content')
print(content)
```
## gemini-robotics-er-1.5-preview Usage
```python
from litellm import api_base
from openai import OpenAI
import os
import base64
client = OpenAI(base_url="http://0.0.0.0:4000", api_key="sk-12345")
base64_image = base64.b64encode(open("closeup-object-on-table-many-260nw-1216144471.webp", "rb").read()).decode()
import json
import re
tools = [{"codeExecution": {}}]
response = client.chat.completions.create(
model="gemini/gemini-robotics-er-1.5-preview",
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "Point to no more than 10 items in the image. The label returned should be an identifying name for the object detected. The answer should follow the json format: [{\"point\": [y, x], \"label\": <label1>}, ...]. The points are in [y, x] format normalized to 0-1000."
},
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}
}
]
}
],
tools=tools
)
# Extract JSON from markdown code block if present
content = response.choices[0].message.content
# Look for triple-backtick JSON block
match = re.search(r'```json\s*(.*?)\s*```', content, re.DOTALL)
if match:
json_str = match.group(1)
else:
json_str = content
try:
data = json.loads(json_str)
print(json.dumps(data, indent=2))
except Exception as e:
print("Error parsing response as JSON:", e)
print("Response content:", content)
```
## Usage - PDF / Videos / etc. Files
### Inline Data (e.g. audio stream)

140
docs/my-website/docs/providers/gmi.md Normal file
View file

@ -0,0 +1,140 @@
# GMI Cloud
## Overview
| Property | Details |
|-------|-------|
| Description | GMI Cloud is a GPU cloud infrastructure provider offering access to top AI models including Claude, GPT, DeepSeek, Gemini, and more through OpenAI-compatible APIs. |
| Provider Route on LiteLLM | `gmi/` |
| Link to Provider Doc | [GMI Cloud Docs ↗](https://docs.gmicloud.ai) |
| Base URL | `https://api.gmi-serving.com/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage), [`/models`](#supported-models) |
<br />
## What is GMI Cloud?
GMI Cloud is a venture-backed digital infrastructure company ($82M+ funding) providing:
- **Top-tier GPU Access**: NVIDIA H100 GPUs for AI workloads
- **Multiple AI Models**: Claude, GPT, DeepSeek, Gemini, Kimi, Qwen, and more
- **OpenAI-Compatible API**: Drop-in replacement for OpenAI SDK
- **Global Infrastructure**: Data centers in US (Colorado) and APAC (Taiwan)
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["GMI_API_KEY"] = "" # your GMI Cloud API key
```
Get your GMI Cloud API key from [console.gmicloud.ai](https://console.gmicloud.ai).
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="GMI Cloud Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["GMI_API_KEY"] = "" # your GMI Cloud API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# GMI Cloud call
response = completion(
model="gmi/deepseek-ai/DeepSeek-V3.2",
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="GMI Cloud Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["GMI_API_KEY"] = "" # your GMI Cloud API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# GMI Cloud call with streaming
response = completion(
model="gmi/anthropic/claude-sonnet-4.5",
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export GMI_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: deepseek-v3
litellm_params:
model: gmi/deepseek-ai/DeepSeek-V3.2
api_key: os.environ/GMI_API_KEY
- model_name: claude-sonnet
litellm_params:
model: gmi/anthropic/claude-sonnet-4.5
api_key: os.environ/GMI_API_KEY
```
## Supported Models
| Model | Model ID | Context Length |
|-------|----------|----------------|
| Claude Opus 4.5 | `gmi/anthropic/claude-opus-4.5` | 409K |
| Claude Sonnet 4.5 | `gmi/anthropic/claude-sonnet-4.5` | 409K |
| Claude Sonnet 4 | `gmi/anthropic/claude-sonnet-4` | 409K |
| Claude Opus 4 | `gmi/anthropic/claude-opus-4` | 409K |
| GPT-5.2 | `gmi/openai/gpt-5.2` | 409K |
| GPT-5.1 | `gmi/openai/gpt-5.1` | 409K |
| GPT-5 | `gmi/openai/gpt-5` | 409K |
| GPT-4o | `gmi/openai/gpt-4o` | 131K |
| GPT-4o-mini | `gmi/openai/gpt-4o-mini` | 131K |
| DeepSeek V3.2 | `gmi/deepseek-ai/DeepSeek-V3.2` | 163K |
| DeepSeek V3 0324 | `gmi/deepseek-ai/DeepSeek-V3-0324` | 163K |
| Gemini 3 Pro | `gmi/google/gemini-3-pro-preview` | 1M |
| Gemini 3 Flash | `gmi/google/gemini-3-flash-preview` | 1M |
| Kimi K2 Thinking | `gmi/moonshotai/Kimi-K2-Thinking` | 262K |
| MiniMax M2.1 | `gmi/MiniMaxAI/MiniMax-M2.1` | 196K |
| Qwen3-VL 235B | `gmi/Qwen/Qwen3-VL-235B-A22B-Instruct-FP8` | 262K |
| GLM-4.7 | `gmi/zai-org/GLM-4.7-FP8` | 202K |
## Supported OpenAI Parameters
GMI Cloud supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID from available models |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
| `response_format` | object | Optional. JSON mode with `{"type": "json_object"}` |
## Additional Resources
- [GMI Cloud Website](https://www.gmicloud.ai)
- [GMI Cloud Documentation](https://docs.gmicloud.ai)
- [GMI Cloud Console](https://console.gmicloud.ai)

2
docs/my-website/docs/providers/openai/text_to_speech.md
View file

@ -46,7 +46,7 @@ os.environ["OPENAI_API_KEY"] = "sk-.."
async def test_async_speech():
speech_file_path = Path(__file__).parent / "speech.mp3"
response = await litellm.aspeech(
response = await aspeech(
model="openai/tts-1",
voice="alloy",
input="the quick brown fox jumped over the lazy dogs",

117
docs/my-website/docs/providers/openrouter.md
View file

@ -93,3 +93,120 @@ response = embedding(
)
print(response)
```
## Image Generation
OpenRouter supports image generation through select models like Google Gemini image generation models. LiteLLM transforms standard image generation requests to OpenRouter's chat completion format.
### Supported Parameters
- `size`: Maps to OpenRouter's `aspect_ratio` format
- `1024x1024` → `1:1` (square)
- `1536x1024` → `3:2` (landscape)
- `1024x1536` → `2:3` (portrait)
- `1792x1024` → `16:9` (wide landscape)
- `1024x1792` → `9:16` (tall portrait)
- `quality`: Maps to OpenRouter's `image_size` format (Gemini models)
- `low` or `standard` → `1K`
- `medium` → `2K`
- `high` or `hd` → `4K`
- `n`: Number of images to generate
### Usage
```python
from litellm import image_generation
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
# Basic image generation
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A beautiful sunset over a calm ocean",
)
print(response)
```
### Advanced Usage with Parameters
```python
from litellm import image_generation
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
# Generate high-quality landscape image
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A serene mountain landscape with a lake",
size="1536x1024", # Landscape format
quality="high", # High quality (4K)
)
# Access the generated image
image_data = response.data[0]
if image_data.b64_json:
# Base64 encoded image
print(f"Generated base64 image: {image_data.b64_json[:50]}...")
elif image_data.url:
# Image URL
print(f"Generated image URL: {image_data.url}")
```
### Using OpenRouter-Specific Parameters
You can also pass OpenRouter-specific parameters directly using `image_config`:
```python
from litellm import image_generation
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A futuristic cityscape at night",
image_config={
"aspect_ratio": "16:9", # OpenRouter native format
"image_size": "4K" # OpenRouter native format
}
)
print(response)
```
### Response Format
The response follows the standard LiteLLM ImageResponse format:
```python
{
"created": 1703658209,
"data": [{
"b64_json": "iVBORw0KGgoAAAANSUhEUgAA...", # Base64 encoded image
"url": None,
"revised_prompt": None
}],
"usage": {
"input_tokens": 10,
"output_tokens": 1290,
"total_tokens": 1300
}
}
```
### Cost Tracking
OpenRouter provides cost information in the response, which LiteLLM automatically tracks:
```python
response = image_generation(
model="openrouter/google/gemini-2.5-flash-image",
prompt="A cute baby sea otter",
)
# Cost is available in the response metadata
print(f"Request cost: ${response._hidden_params['additional_headers']['llm_provider-x-litellm-response-cost']}")
```

535
docs/my-website/docs/providers/sap.md
View file

@ -12,100 +12,340 @@ LiteLLM supports SAP Generative AI Hub's Orchestration Service.
| Supported Endpoints | `/chat/completions`, `/embeddings` |
| API Reference | [SAP AI Core Documentation](https://help.sap.com/docs/sap-ai-core) |
## Prerequisites
Before you begin, ensure you have:
1. **SAP BTP Account** with access to SAP AI Core
2. **AI Core Service Instance** provisioned in your subaccount
3. **Service Key** created for your AI Core instance (this contains your credentials)
4. **Resource Group** with deployed AI models (check with your SAP administrator)
:::tip Where to Find Your Credentials
Your credentials come from the **Service Key** you create in SAP BTP Cockpit:
1. Navigate to your **Subaccount** → **Instances and Subscriptions**
2. Find your **AI Core** instance and click on it
3. Go to **Service Keys** and create one (or use existing)
4. The JSON contains all values needed below
The service key JSON looks like this:
```json
{
"clientid": "sb-abc123...",
"clientsecret": "xyz789...",
"url": "https://myinstance.authentication.eu10.hana.ondemand.com",
"serviceurls": {
"AI_API_URL": "https://api.ai.prod.eu-central-1.aws.ml.hana.ondemand.com"
}
}
```
:::info Resource Group
The resource group is typically configured separately in your AI Core deployment, not in the service key itself. You can set it via the `AICORE_RESOURCE_GROUP` environment variable (defaults to "default").
:::
## Quick Start
### Step 1: Install LiteLLM
```bash
pip install litellm
```
### Step 2: Set Your Credentials
Choose **one** of these authentication methods:
<Tabs>
<TabItem value="service-key" label="Service Key JSON (Recommended)">
The simplest approach - paste your entire service key as a single environment variable. The service key must be wrapped in a `credentials` object:
```bash
export AICORE_SERVICE_KEY='{
"credentials": {
"clientid": "your-client-id",
"clientsecret": "your-client-secret",
"url": "https://<your-instance>.authentication.sap.hana.ondemand.com",
"serviceurls": {
"AI_API_URL": "https://api.ai.<your-region>.aws.ml.hana.ondemand.com"
}
}
}'
export AICORE_RESOURCE_GROUP="default"
```
</TabItem>
<TabItem value="individual" label="Individual Variables">
Alternatively, instead of using the service key above, you could set each credential separately:
```bash
export AICORE_AUTH_URL="https://<your-instance>.authentication.sap.hana.ondemand.com/oauth/token"
export AICORE_CLIENT_ID="your-client-id"
export AICORE_CLIENT_SECRET="your-client-secret"
export AICORE_RESOURCE_GROUP="default"
export AICORE_BASE_URL="https://api.ai.<your-region>.aws.ml.hana.ondemand.com/v2"
```
</TabItem>
</Tabs>
### Step 3: Make Your First Request
```python title="test_sap.py"
from litellm import completion
response = completion(
model="sap/gpt-4o",
messages=[{"role": "user", "content": "Hello from LiteLLM!"}]
)
print(response.choices[0].message.content)
```
Run it:
```bash
python test_sap.py
```
**Expected output:**
```text
Hello! How can I assist you today?
```
### Step 4: Verify Your Setup (Optional)
Test that everything is working with this diagnostic script:
```python title="verify_sap_setup.py"
import os
import litellm
# Enable debug logging to see what's happening
import os
os.environ["LITELLM_LOG"] = "DEBUG"
# Either use AICORE_SERVICE_KEY (contains all credentials including resourcegroup)
# OR use individual variables (all required together)
individual_vars = ["AICORE_AUTH_URL", "AICORE_CLIENT_ID", "AICORE_CLIENT_SECRET", "AICORE_BASE_URL", "AICORE_RESOURCE_GROUP"]
print("=== SAP Gen AI Hub Setup Verification ===\n")
# Check for service key method
if os.environ.get("AICORE_SERVICE_KEY"):
print("✓ Using AICORE_SERVICE_KEY authentication (includes resource group)")
else:
# Check individual variables
missing = [v for v in individual_vars if not os.environ.get(v)]
if missing:
print(f"✗ Missing environment variables: {missing}")
else:
print("✓ Using individual variable authentication")
print(f"✓ Resource group: {os.environ.get('AICORE_RESOURCE_GROUP')}")
# Test API connection
print("\n=== Testing API Connection ===\n")
try:
response = litellm.completion(
model="sap/gpt-4o",
messages=[{"role": "user", "content": "Say 'Connection successful!' and nothing else."}],
max_tokens=20
)
print(f"✓ API Response: {response.choices[0].message.content}")
print("\n🎉 Setup complete! You're ready to use SAP Gen AI Hub with LiteLLM.")
except Exception as e:
print(f"✗ API Error: {e}")
print("\nTroubleshooting tips:")
print(" 1. Verify your service key credentials are correct")
print(" 2. Check that 'gpt-4o' is deployed in your resource group")
print(" 3. Ensure your SAP AI Core instance is running")
```
Run the verification:
```bash
python verify_sap_setup.py
```
**Expected output on success:**
```text
=== SAP Gen AI Hub Setup Verification ===
✓ Using AICORE_SERVICE_KEY authentication
✓ Resource group: default
=== Testing API Connection ===
✓ API Response: Connection successful!
🎉 Setup complete! You're ready to use SAP Gen AI Hub with LiteLLM.
```
## Authentication
SAP Generative AI Hub uses service key authentication. You can provide credentials via:
SAP Generative AI Hub uses OAuth2 service keys for authentication. See [Quick Start](#quick-start) for setup instructions.
1. **Environment variable** - Set `AICORE_SERVICE_KEY` with your service key JSON
2. **Direct parameter** - Pass `api_key` with the service key JSON string
### Environment Variables Reference
```python showLineNumbers title="Environment Variable"
import os
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
| Variable | Required | Description |
|----------|----------|-------------|
| `AICORE_SERVICE_KEY` | Yes* | Complete service key JSON (recommended method) |
| `AICORE_RESOURCE_GROUP` | Yes | Your AI Core resource group name |
| `AICORE_AUTH_URL` | Yes* | OAuth token URL (alternative to service key) |
| `AICORE_CLIENT_ID` | Yes* | OAuth client ID (alternative to service key) |
| `AICORE_CLIENT_SECRET` | Yes* | OAuth client secret (alternative to service key) |
| `AICORE_BASE_URL` | Yes* | AI Core API base URL (alternative to service key) |
*Choose either `AICORE_SERVICE_KEY` OR the individual variables (`AICORE_AUTH_URL`, `AICORE_CLIENT_ID`, `AICORE_CLIENT_SECRET`, `AICORE_BASE_URL`).
## Model Naming Conventions
Understanding model naming is crucial for using SAP Gen AI Hub correctly. The naming pattern differs depending on whether you're using the SDK directly or through the proxy.
### Direct SDK Usage
When calling LiteLLM's SDK directly, you **must** include the `sap/` prefix in the model name:
```python
# Correct - includes sap/ prefix
model="sap/gpt-4o"
model="sap/anthropic--claude-4.5-sonnet"
model="sap/gemini-2.5-pro"
# Incorrect - missing prefix
model="gpt-4o" # ❌ Won't work
```
3. **Environment variables** - Set the following list of credentials in .env file
<pre>
AICORE_AUTH_URL = "https://* * * .authentication.sap.hana.ondemand.com/oauth/token",
AICORE_CLIENT_ID = " *** ",
AICORE_CLIENT_SECRET = " *** ",
AICORE_RESOURCE_GROUP = " *** ",
AICORE_BASE_URL = "https://api.ai.***.cfapps.sap.hana.ondemand.com/v2"
</pre>
## Usage - LiteLLM Python SDK
```python showLineNumbers title="SAP Chat Completion"
from litellm import completion
import os
### Proxy Usage
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
When using the LiteLLM Proxy, you use the **friendly `model_name`** defined in your configuration. The proxy automatically handles the `sap/` prefix routing.
response = completion(
model="sap/gpt-4",
messages=[{"role": "user", "content": "Hello from LiteLLM"}]
```yaml
# In config.yaml, define the mapping
model_list:
- model_name: gpt-4o # ← Use this name in client requests
litellm_params:
model: sap/gpt-4o # ← Proxy handles the sap/ prefix
```
```python
# Client request - no sap/ prefix needed
client.chat.completions.create(
model="gpt-4o", # ✓ Correct for proxy usage
messages=[...]
)
print(response)
```
```python showLineNumbers title="SAP Chat Completion - Streaming"
### Anthropic Models Special Syntax
Anthropic models use a double-dash (`--`) prefix convention:
| Provider | Model Example | LiteLLM Format |
|----------|---------------|----------------|
| OpenAI | GPT-4o | `sap/gpt-4o` |
| Anthropic | Claude 4.5 Sonnet | `sap/anthropic--claude-4.5-sonnet` |
| Google | Gemini 2.5 Pro | `sap/gemini-2.5-pro` |
| Mistral | Mistral Large | `sap/mistral-large` |
### Quick Reference Table
| Usage Type | Model Format | Example |
|------------|--------------|---------|
| Direct SDK | `sap/<model-name>` | `sap/gpt-4o` |
| Direct SDK (Anthropic) | `sap/anthropic--<model>` | `sap/anthropic--claude-4.5-sonnet` |
| Proxy Client | `<friendly-name>` | `gpt-4o` or `claude-sonnet` |
## Using the Python SDK
The LiteLLM Python SDK automatically detects your authentication method. Simply set your environment variables and make requests.
```python showLineNumbers title="Basic Completion"
from litellm import completion
import os
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
# Assumes AICORE_AUTH_URL, AICORE_CLIENT_ID, etc. are set
response = completion(
model="sap/gpt-4",
messages=[{"role": "user", "content": "Hello from LiteLLM"}],
stream=True
model="sap/anthropic--claude-4.5-sonnet",
messages=[{"role": "user", "content": "Explain quantum computing"}]
)
for chunk in response:
print(chunk.choices[0].delta.content or "", end="")
print(response.choices[0].message.content)
```
```python showLineNumbers title="SAP Embedding"
from litellm import embedding
import os
Both authentication methods (individual variables or service key JSON) work automatically - no code changes required.
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
## Using the Proxy Server
result = embedding(
model="sap/text-embedding-3-small",
input="Answer to the ultimate question of life, the universe, and everything is 42")
print(result.data[0])
```
The LiteLLM Proxy provides a unified OpenAI-compatible API for your SAP models.
## Usage - LiteLLM Proxy
### Configuration
Add to your LiteLLM Proxy config:
Create a `config.yaml` file in your project directory with your model mappings and credentials:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: "sap/*"
# OpenAI models
- model_name: gpt-5
litellm_params:
model: "sap/*"
model: sap/gpt-5
general_settings:
master_key: your-proxy-api-key
# Anthropic models (note the double-dash)
- model_name: claude-sonnet
litellm_params:
model: sap/anthropic--claude-4.5-sonnet
- model_name: claude-opus
litellm_params:
model: sap/anthropic--claude-4.5-opus
# Embeddings
- model_name: text-embedding-3-small
litellm_params:
model: sap/text-embedding-3-small
litellm_settings:
drop_params: true
set_verbose: false
request_timeout: 600
num_retries: 2
forward_client_headers_to_llm_api: ["anthropic-version"]
general_settings:
master_key: "sk-1234" # Enter here your desired master key starting with 'sk-'.
# UI Admin is not required but helpful including the management of keys for your team(s). If you are using a database, these parameters are required:
database_url: "Enter you database URL."
UI_USERNAME: "Your desired UI admin account name"
UI_PASSWORD: "Your desired and strong pwd"
# Authentication
environment_variables:
AICORE_SERVICE_KEY: '{"clientid": "...", "clientsecret": "...", ...}'
AICORE_SERVICE_KEY: '{"credentials": {"clientid": "...", "clientsecret": "...", "url": "...", "serviceurls": {"AI_API_URL": "..."}}}'
AICORE_RESOURCE_GROUP: "default"
```
Start the proxy:
### Starting the Proxy
```bash showLineNumbers title="Start Proxy"
litellm --config config.yaml
```
The proxy will start on `http://localhost:4000` by default.
### Making Requests
<Tabs>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Test Request"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "sap/gpt-4",
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
@ -118,11 +358,11 @@ from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-proxy-api-key"
api_key="sk-1234"
)
response = client.chat.completions.create(
model="sap/gpt-4",
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}]
)
print(response.choices[0].message.content)
@ -134,12 +374,14 @@ print(response.choices[0].message.content)
```python showLineNumbers title="LiteLLM SDK"
import os
import litellm
os.environ["LITELLM_PROXY_API_KEY"] = "your-proxy-api-key"
litellm.use_litellm_proxy = True # it is important to set this parameter
os.environ["LITELLM_PROXY_API_KEY"] = "sk-1234"
litellm.use_litellm_proxy = True
response = litellm.completion(
model="sap/gpt-4o",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="http://your-proxy-api-base"
model="claude-sonnet",
messages=[{"content": "Hello, how are you?", "role": "user"}],
api_base="http://localhost:4000"
)
print(response)
@ -148,15 +390,170 @@ print(response)
</TabItem>
</Tabs>
## Supported Parameters
## Features
| Parameter | Description |
|-----------|-------------|
| `temperature` | Controls randomness |
| `max_tokens` | Maximum tokens in response |
| `top_p` | Nucleus sampling |
| `tools` | Function calling tools |
| `tool_choice` | Tool selection behavior |
| `response_format` | Output format (json_object, json_schema) |
| `stream` | Enable streaming |
### Streaming Responses
Stream responses in real-time for better user experience:
```python showLineNumbers title="Streaming Chat Completion"
from litellm import completion
response = completion(
model="sap/gpt-4o",
messages=[{"role": "user", "content": "Count from 1 to 10"}],
stream=True
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="", flush=True)
```
### Structured Output
#### JSON Schema (Recommended)
Use JSON Schema for structured output with strict validation:
```python showLineNumbers title="JSON Schema Response"
from litellm import completion
response = completion(
model="sap/gpt-4o",
messages=[{
"role": "user",
"content": "Generate info about Tokyo"
}],
response_format={
"type": "json_schema",
"json_schema": {
"name": "city_info",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"population": {"type": "number"},
"country": {"type": "string"}
},
"required": ["name", "population", "country"],
"additionalProperties": False
},
"strict": True
}
}
)
print(response.choices[0].message.content)
# Output: {"name":"Tokyo","population":37000000,"country":"Japan"}
```
#### JSON Object Format
For flexible JSON output without schema validation:
```python showLineNumbers title="JSON Object Response"
from litellm import completion
response = completion(
model="sap/gpt-4o",
messages=[{
"role": "user",
"content": "Generate a person object in JSON format with name and age"
}],
response_format={"type": "json_object"}
)
print(response.choices[0].message.content)
```
:::note SAP Platform Requirement
When using `json_object` type, SAP's orchestration service requires the word "json" to appear in your prompt. This ensures explicit intent for JSON formatting. For schema-validated output without this requirement, use `json_schema` instead (recommended).
:::
### Multi-turn Conversations
Maintain conversation context across multiple turns:
```python showLineNumbers title="Multi-turn Conversation"
from litellm import completion
response = completion(
model="sap/gpt-4o",
messages=[
{"role": "user", "content": "My name is Alice"},
{"role": "assistant", "content": "Hello Alice! Nice to meet you."},
{"role": "user", "content": "What is my name?"}
]
)
print(response.choices[0].message.content)
# Output: Your name is Alice.
```
### Embeddings
Generate vector embeddings for semantic search and retrieval:
```python showLineNumbers title="Create Embeddings"
from litellm import embedding
response = embedding(
model="sap/text-embedding-3-small",
input=["Hello world", "Machine learning is fascinating"]
)
print(response.data[0]["embedding"]) # Vector representation
```
## Reference
### Supported Parameters
| Parameter | Type | Description |
|-----------|------|-------------|
| `model` | string | Model identifier (with `sap/` prefix for SDK) |
| `messages` | array | Conversation messages |
| `temperature` | float | Controls randomness (0-2) |
| `max_tokens` | integer | Maximum tokens in response |
| `top_p` | float | Nucleus sampling threshold |
| `stream` | boolean | Enable streaming responses |
| `response_format` | object | Output format (`json_object`, `json_schema`) |
| `tools` | array | Function calling tool definitions |
| `tool_choice` | string/object | Tool selection behavior |
### Supported Models
For the complete and up-to-date list of available models provided by SAP Gen AI Hub, please refer to the [SAP AI Core Generative AI Hub documentation](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/models-and-scenarios-in-generative-ai-hub).
:::info Model Availability
Model availability varies by SAP deployment region and your subscription. Contact your SAP administrator to confirm which models are available in your environment.
:::
### Troubleshooting
**Authentication Errors**
If you receive authentication errors:
1. Verify all required environment variables are set correctly
2. Check that your service key hasn't expired
3. Confirm your resource group has access to the desired models
4. Ensure the `AICORE_AUTH_URL` and `AICORE_BASE_URL` match your SAP region
**Model Not Found**
If a model returns "not found":
1. Verify the model is available in your SAP deployment
2. Check you're using the correct model name format (`sap/` prefix for SDK)
3. Confirm your resource group has access to that specific model
4. For Anthropic models, ensure you're using the `anthropic--` double-dash prefix
**Rate Limiting**
SAP Gen AI Hub enforces rate limits based on your subscription. If you hit limits:
1. Implement exponential backoff retry logic
2. Consider using the proxy's built-in rate limiting features
3. Contact your SAP administrator to review quota allocations

92
docs/my-website/docs/providers/sarvam.md Normal file
View file

@ -0,0 +1,92 @@
# Sarvam.ai
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM supports all the text models from [Sarvam ai](https://docs.sarvam.ai/api-reference-docs/chat/chat-completions)
## Usage
```python
import os
from litellm import completion
# Set your Sarvam API key
os.environ["SARVAM_API_KEY"] = ""
messages = [{"role": "user", "content": "Hello"}]
response = completion(
model="sarvam/sarvam-m",
messages=messages,
)
print(response)
```
## Usage with LiteLLM Proxy Server
Here's how to call a Sarvam.ai model with the LiteLLM Proxy Server
1. **Modify the `config.yaml`:**
```yaml
model_list:
- model_name: my-model
litellm_params:
model: sarvam/<your-model-name> # add sarvam/ prefix to route as Sarvam provider
api_key: api-key # api key to send your model
```
2. **Start the proxy:**
```bash
$ litellm --config /path/to/config.yaml
```
3. **Send a request to LiteLLM Proxy Server:**
<Tabs>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
base_url="http://0.0.0.0:4000" # litellm-proxy-base url
)
response = client.chat.completions.create(
model="my-model",
messages=[
{
"role": "user",
"content": "what llm are you"
}
],
)
print(response)
```
</TabItem>
<TabItem value="curl" label="curl">
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "my-model",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>

53
docs/my-website/docs/providers/stability.md
View file

@ -173,6 +173,14 @@ Stability AI returns images in base64 format. The response is OpenAI-compatible:
Stability AI supports various image editing operations including inpainting, upscaling, outpainting, background removal, and more.
:::info Optional Parameters
**Important:** Different Stability models have different parameter requirements:
- Some models don't require a `prompt` (e.g., upscaling, background removal)
- The `style-transfer` model uses `init_image` and `style_image` instead of `image`
- The `outpaint` model requires numeric parameters (`left`, `right`, `up`, `down`)
LiteLLM automatically handles these differences for you.
:::
### Usage - LiteLLM Python SDK
#### Inpainting (Edit with Mask)
@ -217,11 +225,11 @@ response = image_edit(
creativity=0.3, # 0-0.35, higher = more creative
)
# Fast upscaling - quick upscaling
# Fast upscaling - quick upscaling (no prompt needed)
response = image_edit(
model="stability/stable-fast-upscale-v1:0",
image=open("low_res_image.png", "rb"),
prompt="Quickly upscale this image",
# No prompt required for fast upscale
)
print(response)
```
@ -259,7 +267,7 @@ os.environ['STABILITY_API_KEY'] = "your-api-key"
response = image_edit(
model="stability/stable-image-remove-background-v1:0",
image=open("portrait.png", "rb"),
prompt="Remove the background",
# No prompt required for fast upscale
)
print(response)
```
@ -329,10 +337,29 @@ response = image_edit(
model="stability/stable-image-erase-object-v1:0",
image=open("scene.png", "rb"),
mask=open("object_mask.png", "rb"), # Mask the object to erase
prompt="Remove the object",
# No prompt needed
)
print(response)
```
#### Style Transfer
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Transfer style from one image to another
# Note: Uses init_image (via image param) and style_image
response = image_edit(
model="stability/stable-style-transfer-v1:0",
image=open("content_image.png", "rb"), # Maps to init_image
style_image=open("style_reference.png", "rb"), # Style to apply
fidelity=0.5, # 0-1, balance between content and style
# No prompt needed
)
print(response)
### Supported Image Edit Models
@ -416,10 +443,26 @@ response = image_edit(
image=open("original_image.png", "rb"),
mask=open("mask_image.png", "rb"),
prompt="Add flowers in the masked area",
size="1024x1024",
)
print(response)
```
# Fast upscale without prompt
response = image_edit(
model="bedrock/stability.stable-fast-upscale-v1:0",
image=open("low_res_image.png", "rb"),
)
# Outpaint with numeric parameters
response = image_edit(
model="bedrock/stability.stable-outpaint-v1:0",
image=open("original_image.png", "rb"),
left=100, # Automatically converted to int
right=100,
up=50,
down=50,
)
print(response)
### Supported Bedrock Stability Models

36
docs/my-website/docs/providers/vercel_ai_gateway.md
View file

@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem';
| Provider Route on LiteLLM | `vercel_ai_gateway/` |
| Link to Provider Doc | [Vercel AI Gateway Documentation ↗](https://vercel.com/docs/ai-gateway) |
| Base URL | `https://ai-gateway.vercel.sh/v1` |
| Supported Operations | `/chat/completions`, `/models` |
| Supported Operations | `/chat/completions`, `/embeddings`, `/models` |
<br />
<br />
@ -73,7 +73,7 @@ messages = [{"content": "Hello, how are you?", "role": "user"}]
# Vercel AI Gateway call with streaming
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
model="vercel_ai_gateway/openai/gpt-4o",
messages=messages,
stream=True
)
@ -82,6 +82,33 @@ for chunk in response:
print(chunk)
```
### Embeddings
```python showLineNumbers title="Vercel AI Gateway Embeddings"
import os
from litellm import embedding
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
# Vercel AI Gateway embedding call
response = embedding(
model="vercel_ai_gateway/openai/text-embedding-3-small",
input="Hello world"
)
print(response.data[0]["embedding"][:5]) # Print first 5 dimensions
```
You can also specify the `dimensions` parameter:
```python showLineNumbers title="Vercel AI Gateway Embeddings with Dimensions"
response = embedding(
model="vercel_ai_gateway/openai/text-embedding-3-small",
input=["Hello world", "Goodbye world"],
dimensions=768
)
```
## Usage - LiteLLM Proxy
Add the following to your LiteLLM Proxy configuration file:
@ -97,6 +124,11 @@ model_list:
litellm_params:
model: vercel_ai_gateway/anthropic/claude-4-sonnet
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
- model_name: text-embedding-3-small-gateway
litellm_params:
model: vercel_ai_gateway/openai/text-embedding-3-small
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
```
Start your LiteLLM Proxy server:

320
docs/my-website/docs/providers/vertex.md
View file

@ -14,6 +14,17 @@ import TabItem from '@theme/TabItem';
| Base URL | 1. Regional endpoints<br/>`https://{vertex_location}-aiplatform.googleapis.com/`<br/>2. Global endpoints (limited availability)<br/>`https://aiplatform.googleapis.com/`|
| Supported Operations | [`/chat/completions`](#sample-usage), `/completions`, [`/embeddings`](#embedding-models), [`/audio/speech`](#text-to-speech-apis), [`/fine_tuning`](#fine-tuning-apis), [`/batches`](#batch-apis), [`/files`](#batch-apis), [`/images`](#image-generation-models), [`/rerank`](#rerank-api) |
:::tip Vertex AI vs Gemini API
| Model Format | Provider | Auth Required |
|-------------|----------|---------------|
| `vertex_ai/gemini-2.0-flash` | Vertex AI | GCP credentials + project |
| `gemini-2.0-flash` (no prefix) | Vertex AI | GCP credentials + project |
| `gemini/gemini-2.0-flash` | Gemini API | `GEMINI_API_KEY` (simple API key) |
**If you just want to use an API key** (like OpenAI), use the `gemini/` prefix instead. See [Gemini - Google AI Studio](./gemini.md).
Models without a prefix default to Vertex AI which requires GCP authentication.
:::
<br />
<br />
@ -1390,6 +1401,77 @@ model_list:
### **Workload Identity Federation**
LiteLLM supports [Google Cloud Workload Identity Federation (WIF)](https://cloud.google.com/iam/docs/workload-identity-federation), which allows you to grant on-premises or multi-cloud workloads access to Google Cloud resources without using a service account key. This is the recommended approach for workloads running in other cloud environments (AWS, Azure, etc.) or on-premises.
To use Workload Identity Federation, pass the path to your WIF credentials configuration file via `vertex_credentials`:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="vertex_ai/gemini-1.5-pro",
messages=[{"role": "user", "content": "Hello!"}],
vertex_credentials="/path/to/wif-credentials.json", # 👈 WIF credentials file
vertex_project="your-gcp-project-id",
vertex_location="us-central1"
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: gemini-model
litellm_params:
model: vertex_ai/gemini-1.5-pro
vertex_project: your-gcp-project-id
vertex_location: us-central1
vertex_credentials: /path/to/wif-credentials.json # 👈 WIF credentials file
```
Alternatively, you can create credentials in **LLM Credentials** in the LiteLLM UI and use those to authenticate your models:
```yaml
model_list:
- model_name: gemini-model
litellm_params:
model: vertex_ai/gemini-1.5-pro
vertex_project: your-gcp-project-id
vertex_location: us-central1
litellm_credential_name: my-vertex-wif-credential # 👈 Reference credential stored in UI
```
</TabItem>
</Tabs>
**WIF Credentials File Format**
Your WIF credentials JSON file typically looks like this (for AWS federation):
```json
{
"type": "external_account",
"audience": "//iam.googleapis.com/projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID",
"subject_token_type": "urn:ietf:params:aws:token-type:aws4_request",
"service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/SERVICE_ACCOUNT_EMAIL:generateAccessToken",
"token_url": "https://sts.googleapis.com/v1/token",
"credential_source": {
"environment_id": "aws1",
"region_url": "http://169.254.169.254/latest/meta-data/placement/availability-zone",
"url": "http://169.254.169.254/latest/meta-data/iam/security-credentials",
"regional_cred_verification_url": "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15"
}
}
```
For more details on setting up Workload Identity Federation, see [Google Cloud WIF documentation](https://cloud.google.com/iam/docs/workload-identity-federation).
### **Environment Variables**
You can set:
@ -1886,6 +1968,244 @@ assert isinstance(
```
## Media Resolution Control (Images & Videos)
For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types.
**Supported `detail` values:**
- `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos)
- `"medium"` - Maps to `media_resolution: "medium"`
- `"high"` - Maps to `media_resolution: "high"` (1120 tokens for images)
- `"ultra_high"` - Maps to `media_resolution: "ultra_high"`
- `"auto"` or `None` - Model decides optimal resolution (no `media_resolution` set)
**Usage Examples:**
<Tabs>
<TabItem value="images" label="Images">
```python
from litellm import completion
messages = [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": "https://example.com/chart.png",
"detail": "high" # High resolution for detailed chart analysis
}
},
{
"type": "text",
"text": "Analyze this chart"
},
{
"type": "image_url",
"image_url": {
"url": "https://example.com/icon.png",
"detail": "low" # Low resolution for simple icon
}
}
]
}
]
response = completion(
model="vertex_ai/gemini-3-pro-preview",
messages=messages,
)
```
</TabItem>
<TabItem value="videos" label="Videos with Files">
```python
from litellm import completion
messages = [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Analyze this video"
},
{
"type": "file",
"file": {
"file_id": "gs://my-bucket/video.mp4",
"format": "video/mp4",
"detail": "high" # High resolution for detailed video analysis
}
}
]
}
]
response = completion(
model="vertex_ai/gemini-3-pro-preview",
messages=messages,
)
```
</TabItem>
</Tabs>
:::info
**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types, and is only available for Gemini 3+ models.
:::
## Video Metadata Control
For Gemini 3+ models, LiteLLM supports fine-grained video processing control through the `video_metadata` field. This allows you to specify frame extraction rates and time ranges for video analysis.
**Supported `video_metadata` parameters:**
| Parameter | Type | Description | Example |
|-----------|------|-------------|---------|
| `fps` | Number | Frame extraction rate (frames per second) | `5` |
| `start_offset` | String | Start time for video clip processing | `"10s"` |
| `end_offset` | String | End time for video clip processing | `"60s"` |
:::note
**Field Name Conversion:** LiteLLM automatically converts snake_case field names to camelCase for the Gemini API:
- `start_offset` → `startOffset`
- `end_offset` → `endOffset`
- `fps` remains unchanged
:::
:::warning
- **Gemini 3+ Only:** This feature is only available for Gemini 3.0 and newer models
- **Video Files Recommended:** While `video_metadata` is designed for video files, error handling for other media types is delegated to the Vertex AI API
- **File Formats Supported:** Works with `gs://`, `https://`, and base64-encoded video files
:::
**Usage Examples:**
<Tabs>
<TabItem value="basic" label="Basic Video Metadata">
```python
from litellm import completion
response = completion(
model="vertex_ai/gemini-3-pro-preview",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this video clip"},
{
"type": "file",
"file": {
"file_id": "gs://my-bucket/video.mp4",
"format": "video/mp4",
"video_metadata": {
"fps": 5, # Extract 5 frames per second
"start_offset": "10s", # Start from 10 seconds
"end_offset": "60s" # End at 60 seconds
}
}
}
]
}
]
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="combined" label="Combined with Detail">
```python
from litellm import completion
response = completion(
model="vertex_ai/gemini-3-pro-preview",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "Provide detailed analysis of this video segment"},
{
"type": "file",
"file": {
"file_id": "https://example.com/presentation.mp4",
"format": "video/mp4",
"detail": "high", # High resolution for detailed analysis
"video_metadata": {
"fps": 10, # Extract 10 frames per second
"start_offset": "30s", # Start from 30 seconds
"end_offset": "90s" # End at 90 seconds
}
}
}
]
}
]
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: gemini-3-pro
litellm_params:
model: vertex_ai/gemini-3-pro-preview
vertex_project: your-project
vertex_location: us-central1
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Make request
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-d '{
"model": "gemini-3-pro",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "Analyze this video clip"},
{
"type": "file",
"file": {
"file_id": "gs://my-bucket/video.mp4",
"format": "video/mp4",
"detail": "high",
"video_metadata": {
"fps": 5,
"start_offset": "10s",
"end_offset": "60s"
}
}
}
]
}
]
}'
```
</TabItem>
</Tabs>
## Usage - PDF / Videos / Audio etc. Files

15
docs/my-website/docs/proxy/caching.md
View file

@ -282,6 +282,10 @@ Set either `REDIS_URL` or the `REDIS_HOST` in your os environment, to enable cac
```
**Additional kwargs**
:::info
Use `REDIS_*` environment variables to configure all Redis client library parameters. This is the suggested mechanism for toggling Redis settings as it automatically maps environment variables to Redis client kwargs.
:::
You can pass in any additional redis.Redis arg, by storing the variable + value in your os
environment, like this:
@ -289,6 +293,17 @@ environment, like this:
REDIS_<redis-kwarg-name> = ""
```
For example:
```shell
REDIS_SSL = "True"
REDIS_SSL_CERT_REQS = "None"
REDIS_CONNECTION_POOL_KWARGS = '{"max_connections": 20}'
```
:::warning
**Note**: For non-string Redis parameters (like integers, booleans, or complex objects), avoid using `REDIS_*` environment variables as they may fail during Redis client initialization. Instead, use `cache_kwargs` in your router configuration for such parameters.
:::
[**See how it's read from the environment**](https://github.com/BerriAI/litellm/blob/4d7ff1b33b9991dcf38d821266290631d9bcd2dd/litellm/_redis.py#L40)
#### Step 3: Run proxy with config

41
docs/my-website/docs/proxy/call_hooks.md
View file

@ -19,6 +19,7 @@ import Image from '@theme/IdealImage';
| `async_post_call_success_hook` | Modify outgoing response (non-streaming) | After successful LLM API call, for non-streaming responses |
| `async_post_call_failure_hook` | Transform error responses sent to clients | After failed LLM API call |
| `async_post_call_streaming_hook` | Modify outgoing response (streaming) | After successful LLM API call, for streaming responses |
| `async_post_call_response_headers_hook` | Inject custom HTTP response headers | After LLM API call (both success and failure) |
See a complete example with our [parallel request rate limiter](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/parallel_request_limiter.py)
@ -115,6 +116,18 @@ class MyCustomHandler(CustomLogger): # https://docs.litellm.ai/docs/observabilit
async for item in response:
yield item
async def async_post_call_response_headers_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
request_headers: Optional[Dict[str, str]] = None,
) -> Optional[Dict[str, str]]:
"""
Inject custom headers into HTTP response (runs for both success and failure).
"""
return {"x-custom-header": "custom-value"}
proxy_handler_instance = MyCustomHandler()
```
@ -389,3 +402,31 @@ proxy_handler_instance = MyErrorTransformer()
```
**Result:** Clients receive `"Your prompt is too long..."` instead of `"ContextWindowExceededError: Prompt exceeds context window"`.
## Advanced - Inject Custom HTTP Response Headers
Use `async_post_call_response_headers_hook` to inject custom HTTP headers into responses. This hook runs for **both successful and failed** LLM API calls.
```python
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy.proxy_server import UserAPIKeyAuth
from typing import Any, Dict, Optional
class CustomHeaderLogger(CustomLogger):
def __init__(self):
super().__init__()
async def async_post_call_response_headers_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
request_headers: Optional[Dict[str, str]] = None,
) -> Optional[Dict[str, str]]:
"""
Inject custom headers into all responses (success and failure).
"""
return {"x-custom-header": "custom-value"}
proxy_handler_instance = CustomHeaderLogger()
```

31
docs/my-website/docs/proxy/cli_sso.md
View file

@ -28,6 +28,37 @@ EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
:::
### Configuration
#### JWT Token Expiration
By default, CLI authentication tokens expire after **24 hours**. You can customize this expiration time by setting the `LITELLM_CLI_JWT_EXPIRATION_HOURS` environment variable when starting your LiteLLM Proxy:
```bash
# Set CLI JWT tokens to expire after 48 hours
export LITELLM_CLI_JWT_EXPIRATION_HOURS=48
export EXPERIMENTAL_UI_LOGIN="True"
litellm --config config.yaml
```
Or in a single command:
```bash
LITELLM_CLI_JWT_EXPIRATION_HOURS=48 EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
```
**Examples:**
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=12` - Tokens expire after 12 hours
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=168` - Tokens expire after 7 days (168 hours)
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=720` - Tokens expire after 30 days (720 hours)
:::tip
You can check your current token's age and expiration status using:
```bash
litellm-proxy whoami
```
:::
### Steps
1. **Install the CLI**

43
docs/my-website/docs/proxy/config_settings.md
View file

@ -178,6 +178,7 @@ router_settings:
| turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data [Proxy Logging](logging) |
| modify_params | boolean | If true, allows modifying the parameters of the request before it is sent to the LLM provider |
| enable_preview_features | boolean | If true, enables preview features - e.g. Azure O1 Models with streaming support.|
| LITELLM_DISABLE_STOP_SEQUENCE_LIMIT | Disable validation for stop sequence limit (default: 4) |
| redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) |
| mcp_aliases | object | Maps friendly aliases to MCP server names for easier tool access. Only the first alias for each server is used. [MCP Aliases](../mcp#mcp-aliases) |
| langfuse_default_tags | array of strings | Default tags for Langfuse Logging. Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields as tags. [Further docs](./logging#litellm-specific-tags-on-langfuse---cache_hit-cache_key) |
@ -339,7 +340,7 @@ router_settings:
| stream_timeout | Optional[float] | The default timeout for a streaming request. If not set, the 'timeout' value is used. |
| debug_level | Literal["DEBUG", "INFO"] | The debug level for the logging library in the router. Defaults to "INFO". |
| client_ttl | int | Time-to-live for cached clients in seconds. Defaults to 3600. |
| cache_kwargs | dict | Additional keyword arguments for the cache initialization. |
| cache_kwargs | dict | Additional keyword arguments for the cache initialization. Use this for non-string Redis parameters that may fail when set via `REDIS_*` environment variables. |
| routing_strategy_args | dict | Additional keyword arguments for the routing strategy - e.g. lowest latency routing default ttl |
| model_group_alias | dict | Model group alias mapping. E.g. `{"claude-3-haiku": "claude-3-haiku-20240229"}` |
| num_retries | int | Number of retries for a request. Defaults to 3. |
@ -397,6 +398,7 @@ router_settings:
| AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024
| ANTHROPIC_API_KEY | API key for Anthropic service
| ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
| ANTHROPIC_TOKEN_COUNTING_BETA_VERSION | Beta version header for Anthropic token counting API. Default is `token-counting-2024-11-01`
| AWS_ACCESS_KEY_ID | Access Key ID for AWS services
| AWS_BATCH_ROLE_ARN | ARN of the AWS IAM role for batch operations
| AWS_DEFAULT_REGION | Default AWS region for service interactions when AWS_REGION is not set
@ -412,6 +414,8 @@ router_settings:
| AWS_WEB_IDENTITY_TOKEN | Web identity token for AWS
| AWS_WEB_IDENTITY_TOKEN_FILE | Path to file containing web identity token for AWS
| AZURE_API_VERSION | Version of the Azure API being used
| AZURE_AI_API_BASE | Base URL for Azure AI services (e.g., Azure AI Anthropic)
| AZURE_AI_API_KEY | API key for Azure AI services (e.g., Azure AI Anthropic)
| AZURE_AUTHORITY_HOST | Azure authority host URL
| AZURE_CERTIFICATE_PASSWORD | Password for Azure OpenAI certificate
| AZURE_CLIENT_ID | Client ID for Azure services
@ -448,9 +452,19 @@ router_settings:
| BERRISPEND_ACCOUNT_ID | Account ID for BerriSpend service
| BRAINTRUST_API_KEY | API key for Braintrust integration
| BRAINTRUST_API_BASE | Base URL for Braintrust API. Default is https://api.braintrustdata.com/v1
| BRAINTRUST_MOCK | Enable mock mode for Braintrust integration testing. When set to true, intercepts Braintrust API calls and returns mock responses without making actual network calls. Default is false
| BRAINTRUST_MOCK_LATENCY_MS | Mock latency in milliseconds for Braintrust API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| CACHED_STREAMING_CHUNK_DELAY | Delay in seconds for cached streaming chunks. Default is 0.02
| CHATGPT_API_BASE | Base URL for ChatGPT API. Default is https://chatgpt.com/backend-api/codex
| CHATGPT_AUTH_FILE | Filename for ChatGPT authentication data. Default is "auth.json"
| CHATGPT_DEFAULT_INSTRUCTIONS | Default system instructions for ChatGPT provider
| CHATGPT_ORIGINATOR | Originator identifier for ChatGPT API requests. Default is "codex_cli_rs"
| CHATGPT_TOKEN_DIR | Directory to store ChatGPT authentication tokens. Default is "~/.config/litellm/chatgpt"
| CHATGPT_USER_AGENT | Custom user agent string for ChatGPT API requests
| CHATGPT_USER_AGENT_SUFFIX | Suffix to append to the ChatGPT user agent string
| CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI
| CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
| CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours. Can also be set via LITELLM_CLI_JWT_EXPIRATION_HOURS
| CLOUDZERO_API_KEY | CloudZero API key for authentication
| CLOUDZERO_CONNECTION_ID | CloudZero connection ID for data submission
| CLOUDZERO_EXPORT_INTERVAL_MINUTES | Interval in minutes for CloudZero data export operations
@ -493,12 +507,15 @@ router_settings:
| DD_AGENT_HOST | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API
| DD_AGENT_PORT | Port of DataDog agent for log intake. Default is 10518
| DD_API_KEY | API key for Datadog integration
| DD_APP_KEY | Application key for Datadog Cost Management integration. Required along with DD_API_KEY for cost metrics
| DD_SITE | Site URL for Datadog (e.g., datadoghq.com)
| DD_SOURCE | Source identifier for Datadog logs
| DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE | Resource name for Datadog tracing of streaming chunk yields. Default is "streaming.chunk.yield"
| DD_ENV | Environment identifier for Datadog logs. Only supported for `datadog_llm_observability` callback
| DD_SERVICE | Service identifier for Datadog logs. Defaults to "litellm-server"
| DD_VERSION | Version identifier for Datadog logs. Defaults to "unknown"
| DATADOG_MOCK | Enable mock mode for Datadog integration testing. When set to true, intercepts Datadog API calls and returns mock responses without making actual network calls. Default is false
| DATADOG_MOCK_LATENCY_MS | Mock latency in milliseconds for Datadog API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| DEBUG_OTEL | Enable debug mode for OpenTelemetry
| DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3
| DEFAULT_A2A_AGENT_TIMEOUT | Default timeout in seconds for A2A (Agent-to-Agent) protocol requests. Default is 6000
@ -600,9 +617,12 @@ router_settings:
| GALILEO_USERNAME | Username for Galileo authentication
| GOOGLE_SECRET_MANAGER_PROJECT_ID | Project ID for Google Secret Manager
| GCS_BUCKET_NAME | Name of the Google Cloud Storage bucket
| GCS_MOCK | Enable mock mode for GCS integration testing. When set to true, intercepts GCS API calls and returns mock responses without making actual network calls. Default is false
| GCS_MOCK_LATENCY_MS | Mock latency in milliseconds for GCS API calls when mock mode is enabled. Simulates network round-trip time. Default is 150ms
| GCS_PATH_SERVICE_ACCOUNT | Path to the Google Cloud service account JSON file
| GCS_FLUSH_INTERVAL | Flush interval for GCS logging (in seconds). Specify how often you want a log to be sent to GCS. **Default is 20 seconds**
| GCS_BATCH_SIZE | Batch size for GCS logging. Specify after how many logs you want to flush to GCS. If `BATCH_SIZE` is set to 10, logs are flushed every 10 logs. **Default is 2048**
| GCS_USE_BATCHED_LOGGING | Enable batched logging for GCS. When enabled (default), multiple log payloads are combined into single GCS object uploads (NDJSON format), dramatically reducing API calls. When disabled, sends each log individually as separate GCS objects (legacy behavior). **Default is true**
| GCS_PUBSUB_TOPIC_ID | PubSub Topic ID to send LiteLLM SpendLogs to.
| GCS_PUBSUB_PROJECT_ID | PubSub Project ID to send LiteLLM SpendLogs to.
| GENERIC_AUTHORIZATION_ENDPOINT | Authorization endpoint for generic OAuth providers
@ -624,6 +644,10 @@ router_settings:
| GENERIC_USERINFO_ENDPOINT | Endpoint to fetch user information in generic OAuth
| GENERIC_LOGGER_ENDPOINT | Endpoint URL for the Generic Logger callback to send logs to
| GENERIC_LOGGER_HEADERS | JSON string of headers to include in Generic Logger callback requests
| GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE | Default LiteLLM role to assign when no role mapping matches in generic SSO. Used with GENERIC_ROLE_MAPPINGS_ROLES
| GENERIC_ROLE_MAPPINGS_GROUP_CLAIM | The claim/attribute name in the SSO token that contains the user's groups. Used for role mapping
| GENERIC_ROLE_MAPPINGS_ROLES | Python dict string mapping LiteLLM roles to SSO group names. Example: `{"proxy_admin": ["admin-group"], "internal_user": ["users"]}`
| GENERIC_USER_ROLE_MAPPINGS | Alternative to GENERIC_ROLE_MAPPINGS_ROLES for configuring user role mappings from SSO
| GEMINI_API_BASE | Base URL for Gemini API. Default is https://generativelanguage.googleapis.com
| GALILEO_BASE_URL | Base URL for Galileo platform
| GALILEO_PASSWORD | Password for Galileo authentication
@ -660,6 +684,8 @@ router_settings:
| HCP_VAULT_CERT_ROLE | Role for [Hashicorp Vault Secret Manager Auth](../secret.md#hashicorp-vault)
| HELICONE_API_KEY | API key for Helicone service
| HELICONE_API_BASE | Base URL for Helicone service, defaults to `https://api.helicone.ai`
| HELICONE_MOCK | Enable mock mode for Helicone integration testing. When set to true, intercepts Helicone API calls and returns mock responses without making actual network calls. Default is false
| HELICONE_MOCK_LATENCY_MS | Mock latency in milliseconds for Helicone API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| HOSTNAME | Hostname for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog)
| HOURS_IN_A_DAY | Hours in a day for calculation purposes. Default is 24
| HIDDENLAYER_API_BASE | Base URL for HiddenLayer API. Defaults to `https://api.hiddenlayer.ai`
@ -685,6 +711,8 @@ router_settings:
| LANGFUSE_FLUSH_INTERVAL | Interval for flushing Langfuse logs
| LANGFUSE_TRACING_ENVIRONMENT | Environment for Langfuse tracing
| LANGFUSE_HOST | Host URL for Langfuse service
| LANGFUSE_MOCK | Enable mock mode for Langfuse integration testing. When set to true, intercepts Langfuse API calls and returns mock responses without making actual network calls. Default is false
| LANGFUSE_MOCK_LATENCY_MS | Mock latency in milliseconds for Langfuse API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| LANGFUSE_PUBLIC_KEY | Public key for Langfuse authentication
| LANGFUSE_RELEASE | Release version of Langfuse integration
| LANGFUSE_SECRET_KEY | Secret key for Langfuse authentication
@ -696,6 +724,8 @@ router_settings:
| LANGSMITH_PROJECT | Project name for Langsmith integration
| LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging
| LANGSMITH_TENANT_ID | Tenant ID for Langsmith multi-tenant deployments
| LANGSMITH_MOCK | Enable mock mode for Langsmith integration testing. When set to true, intercepts Langsmith API calls and returns mock responses without making actual network calls. Default is false
| LANGSMITH_MOCK_LATENCY_MS | Mock latency in milliseconds for Langsmith API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| LANGTRACE_API_KEY | API key for Langtrace service
| LASSO_API_BASE | Base URL for Lasso API
| LASSO_API_KEY | API key for Lasso service
@ -707,8 +737,10 @@ router_settings:
| LITERAL_API_URL | API URL for Literal service
| LITERAL_BATCH_SIZE | Batch size for Literal operations
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
| LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
| LITELLM_DD_LLM_OBS_PORT | Port for Datadog LLM Observability agent. Default is 8126
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
| LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
@ -744,6 +776,7 @@ router_settings:
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LOGFIRE_TOKEN | Token for Logfire logging service
| LOGFIRE_BASE_URL | Base URL for Logfire logging service (useful for self hosted deployments)
| LOGGING_WORKER_CONCURRENCY | Maximum number of concurrent coroutine slots for the logging worker on the asyncio event loop. Default is 100. Setting too high will flood the event loop with logging tasks which will lower the overall latency of the requests.
| LOGGING_WORKER_MAX_QUEUE_SIZE | Maximum size of the logging worker queue. When the queue is full, the worker aggressively clears tasks to make room instead of dropping logs. Default is 50,000
| LOGGING_WORKER_MAX_TIME_PER_COROUTINE | Maximum time in seconds allowed for each coroutine in the logging worker before timing out. Default is 20.0
@ -754,6 +787,7 @@ router_settings:
| LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS | Cooldown time in seconds before allowing another aggressive clear operation when the queue is full. Default is 0.5
| MAX_STRING_LENGTH_PROMPT_IN_DB | Maximum length for strings in spend logs when sanitizing request bodies. Strings longer than this will be truncated. Default is 1000
| MAX_IN_MEMORY_QUEUE_FLUSH_COUNT | Maximum count for in-memory queue flush operations. Default is 1000
| MAX_IMAGE_URL_DOWNLOAD_SIZE_MB | Maximum size in MB for downloading images from URLs. Prevents memory issues from downloading very large images. Images exceeding this limit will be rejected before download. Set to 0 to completely disable image URL handling (all image_url requests will be blocked). Default is 50MB (matching [OpenAI's limit](https://platform.openai.com/docs/guides/images-vision?api-mode=chat#image-input-requirements))
| MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the long side of high-resolution images. Default is 2000
| MAX_REDIS_BUFFER_DEQUEUE_COUNT | Maximum count for Redis buffer dequeue operations. Default is 100
| MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the short side of high-resolution images. Default is 768
@ -791,6 +825,7 @@ router_settings:
| OPENAI_BASE_URL | Base URL for OpenAI API
| OPENAI_API_BASE | Base URL for OpenAI API. Default is https://api.openai.com/
| OPENAI_API_KEY | API key for OpenAI services
| OPENAI_CHATGPT_API_BASE | Alternative to CHATGPT_API_BASE. Base URL for ChatGPT API
| OPENAI_FILE_SEARCH_COST_PER_1K_CALLS | Cost per 1000 calls for OpenAI file search. Default is 0.0025
| OPENAI_ORGANIZATION | Organization identifier for OpenAI
| OPENID_BASE_URL | Base URL for OpenID Connect services
@ -801,6 +836,7 @@ router_settings:
| OPENMETER_EVENT_TYPE | Type of events sent to OpenMeter
| ONYX_API_BASE | Base URL for Onyx Security AI Guard service (defaults to https://ai-guard.onyx.security)
| ONYX_API_KEY | API key for Onyx Security AI Guard service
| ONYX_TIMEOUT | Timeout in seconds for Onyx Guard server requests. Default is 10
| OTEL_ENDPOINT | OpenTelemetry endpoint for traces
| OTEL_EXPORTER_OTLP_ENDPOINT | OpenTelemetry endpoint for traces
| OTEL_ENVIRONMENT_NAME | Environment name for OpenTelemetry
@ -824,6 +860,8 @@ router_settings:
| POD_NAME | Pod name for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog) as `POD_NAME`
| POSTHOG_API_KEY | API key for PostHog analytics integration
| POSTHOG_API_URL | Base URL for PostHog API (defaults to https://us.i.posthog.com)
| POSTHOG_MOCK | Enable mock mode for PostHog integration testing. When set to true, intercepts PostHog API calls and returns mock responses without making actual network calls. Default is false
| POSTHOG_MOCK_LATENCY_MS | Mock latency in milliseconds for PostHog API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| PREDIBASE_API_BASE | Base URL for Predibase API
| PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
| PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service
@ -861,9 +899,12 @@ router_settings:
| ROUTER_MAX_FALLBACKS | Maximum number of fallbacks for router. Default is 5
| RUNWAYML_DEFAULT_API_VERSION | Default API version for RunwayML service. Default is "2024-11-06"
| RUNWAYML_POLLING_TIMEOUT | Timeout in seconds for RunwayML image generation polling. Default is 600 (10 minutes)
| S3_VECTORS_DEFAULT_DIMENSION | Default vector dimension for S3 Vectors RAG ingestion. Default is 1024
| S3_VECTORS_DEFAULT_DISTANCE_METRIC | Default distance metric for S3 Vectors RAG ingestion. Options: "cosine", "euclidean". Default is "cosine"
| SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours)
| SEPARATE_HEALTH_APP | If set to '1', runs health endpoints on a separate ASGI app and port. Default: '0'.
| SEPARATE_HEALTH_PORT | Port for the separate health endpoints app. Only used if SEPARATE_HEALTH_APP=1. Default: 4001.
| SUPERVISORD_STOPWAITSECS | Upper bound timeout in seconds for graceful shutdown when SEPARATE_HEALTH_APP=1. Default: 3600 (1 hour).
| SERVER_ROOT_PATH | Root path for the server application
| SEND_USER_API_KEY_ALIAS | Flag to send user API key alias to Zscaler AI Guard. Default is False
| SEND_USER_API_KEY_TEAM_ID | Flag to send user API key team ID to Zscaler AI Guard. Default is False

68
docs/my-website/docs/proxy/custom_pricing.md
View file

@ -9,7 +9,6 @@ LiteLLM provides flexible cost tracking and pricing customization for all LLM pr
- **Custom Pricing** - Override default model costs or set pricing for custom models
- **Cost Per Token** - Track costs based on input/output tokens (most common)
- **Cost Per Second** - Track costs based on runtime (e.g., Sagemaker)
- **Zero-Cost Models** - Bypass budget checks for free/on-premises models by setting costs to 0
- **[Provider Discounts](./provider_discounts.md)** - Apply percentage-based discounts to specific providers
- **[Provider Margins](./provider_margins.md)** - Add fees/margins to LLM costs for internal billing
- **Base Model Mapping** - Ensure accurate cost tracking for Azure deployments
@ -107,51 +106,6 @@ There are other keys you can use to specify costs for different scenarios and mo
These keys evolve based on how new models handle multimodality. The latest version can be found at [https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
## Zero-Cost Models (Bypass Budget Checks)
**Use Case**: You have on-premises or free models that should be accessible even when users exceed their budget limits.
**Solution** ✅: Set both `input_cost_per_token` and `output_cost_per_token` to `0` (explicitly) to bypass all budget checks for that model.
:::info
When a model is configured with zero cost, LiteLLM will automatically skip ALL budget checks (user, team, team member, end-user, organization, and global proxy budget) for requests to that model.
**Important**: Both costs must be **explicitly set to 0**. If costs are `null` or undefined, the model will be treated as having cost and budget checks will apply.
:::
### Configuration Example
```yaml
model_list:
# On-premises model - free to use
- model_name: on-prem-llama
litellm_params:
model: ollama/llama3
api_base: http://localhost:11434
model_info:
input_cost_per_token: 0 # 👈 Explicitly set to 0
output_cost_per_token: 0 # 👈 Explicitly set to 0
# Paid cloud model - budget checks apply
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
# No model_info - uses default pricing from cost map
```
### Behavior
With the above configuration:
- **User over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
- **Team over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
- **End-user over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
This ensures your free/on-premises models remain accessible regardless of budget constraints, while paid models are still properly governed.
## Set 'base_model' for Cost Tracking (e.g. Azure deployments)
**Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking
@ -173,6 +127,28 @@ model_list:
base_model: azure/gpt-4-1106-preview
```
### OpenAI Models with Dated Versions
`base_model` is also useful when OpenAI returns a dated model name in the response that differs from your configured model name.
**Example**: You configure custom pricing for `gpt-4o-mini-audio-preview`, but OpenAI returns `gpt-4o-mini-audio-preview-2024-12-17` in the response. Since LiteLLM uses the response model name for pricing lookup, your custom pricing won't be applied.
**Solution** ✅: Set `base_model` to the key you want LiteLLM to use for pricing lookup.
```yaml
model_list:
- model_name: my-audio-model
litellm_params:
model: openai/gpt-4o-mini-audio-preview
api_key: os.environ/OPENAI_API_KEY
model_info:
base_model: gpt-4o-mini-audio-preview # 👈 Used for pricing lookup
input_cost_per_token: 0.0000006
output_cost_per_token: 0.0000024
input_cost_per_audio_token: 0.00001
output_cost_per_audio_token: 0.00002
```
## Debugging

57
docs/my-website/docs/proxy/customer_usage.md
View file

@ -22,19 +22,22 @@ Customer Usage enables you to track spend and usage for individual customers (en
## How to Track Spend
Track customer spend by including a `user` field in your API requests. The customer ID will be automatically tracked and associated with all spend from that request.
Track customer spend by including a `user` field in your API requests or by passing a customer ID header. The customer ID will be automatically tracked and associated with all spend from that request.
### Example using cURL
<Tabs>
<TabItem value="body" label="Request Body" default>
### Using Request Body
Make a `/chat/completions` call with the `user` field containing your customer ID:
```bash showLineNumbers title="Track spend with customer ID"
```bash showLineNumbers title="Track spend with customer ID in body"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "gpt-3.5-turbo",
"user": "customer-123", # 👈 CUSTOMER ID
"user": "customer-123",
"messages": [
{
"role": "user",
@ -44,7 +47,49 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
}'
```
The customer ID (`customer-123`) will be automatically upserted into the database with the new spend. If the customer ID already exists, spend will be incremented.
</TabItem>
<TabItem value="header" label="Request Header">
### Using Request Headers
You can also pass the customer ID via HTTP headers. This is useful for tools that support custom headers but don't allow modifying the request body (like Claude Code with `ANTHROPIC_CUSTOM_HEADERS`).
LiteLLM automatically recognizes these standard headers (no configuration required):
- `x-litellm-customer-id`
- `x-litellm-end-user-id`
```bash showLineNumbers title="Track spend with customer ID in header"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--header 'x-litellm-customer-id: customer-123' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
]
}'
```
#### Using with Claude Code
Claude Code supports custom headers via the `ANTHROPIC_CUSTOM_HEADERS` environment variable. Set it to pass your customer ID:
```bash title="Configure Claude Code with customer tracking"
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/v1/messages"
export ANTHROPIC_API_KEY="sk-1234"
export ANTHROPIC_CUSTOM_HEADERS="x-litellm-customer-id: my-customer-id"
```
Now all requests from Claude Code will automatically track spend under `my-customer-id`.
</TabItem>
</Tabs>
The customer ID will be automatically upserted into the database with the new spend. If the customer ID already exists, spend will be incremented.
### Example using OpenWebUI

106
docs/my-website/docs/proxy/deleted_keys_teams.md Normal file
View file

@ -0,0 +1,106 @@
import Image from '@theme/IdealImage';
# Deleted Keys & Teams Audit Logs
<Image img={require('../../img/ui_deleted_keys_table.png')} />
View deleted API keys and teams along with their spend and budget information at the time of deletion for auditing and compliance purposes.
## Overview
The Deleted Keys & Teams feature provides a comprehensive audit trail for deleted entities in your LiteLLM proxy. This feature was implemented to easily allow audits of which key or team was deleted along with the spend/budget at the time of deletion.
When a key or team is deleted, LiteLLM automatically captures:
- **Deletion timestamp** - When the entity was deleted
- **Deleted by** - Who performed the deletion action
- **Spend at deletion** - The total spend accumulated at the time of deletion
- **Original budget** - The budget that was set for the entity before deletion
- **Entity details** - Key or team identification information
This information is preserved even after deletion, allowing you to maintain accurate financial records and audit trails for compliance purposes.
## Viewing Deleted Keys
### Step 1: Navigate to API Keys Page
Navigate to the API Keys page in the LiteLLM UI:
```
http://localhost:4000/ui/?login=success&page=api-keys
```
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/73b97ba9-0ab5-4140-aee2-05fa90463461/ascreenshot_5e6d9f05d452405c83d7a368349d087d_text_export.jpeg)
### Step 2: Access Logs Section
Click on the "Logs" menu item in the navigation.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/73b97ba9-0ab5-4140-aee2-05fa90463461/ascreenshot_8ebab354b1e542e59e1082e519927edd_text_export.jpeg)
### Step 3: View Deleted Keys
Click on "Deleted Keys" to view the table of all deleted API keys.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/00668558-9326-4a6f-8e87-159d54b17a72/ascreenshot_d0e50e49e9aa43d4a22ada6f12a78b12_text_export.jpeg)
### Step 4: Review Deletion Information
The Deleted Keys table includes comprehensive information about each deleted key:
- **When** the key was deleted (timestamp)
- **Who** deleted the key (user/admin information)
- **Key identification** details
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/8538f7c4-634e-44c8-8d7d-fafbd6da0b02/ascreenshot_6b73f9c6a52d4e40a2368ef441cf6c8f_text_export.jpeg)
### Step 5: View Financial Information
The table also displays financial information captured at the time of deletion:
- **Spend at deletion** - Total spend accumulated when the key was deleted
- **Original budget** - The budget limit that was set for the key
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/f8b03850-b17c-490c-a507-c3b0b6c050ab/ascreenshot_070b139f111844bba38fbed8835b097b_text_export.jpeg)
## Viewing Deleted Teams
### Step 1: Access Deleted Teams
From the Logs section, click on "Deleted Teams" to view all deleted teams.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/716ce26f-09af-4a6d-99c5-921d6b6a8555/ascreenshot_d36c16f1cf894340aa8bc20ada5922ac_text_export.jpeg)
### Step 2: Review Team Deletion Information
The Deleted Teams table provides detailed information about each deleted team:
- **When** the team was deleted (timestamp)
- **Who** deleted the team (user/admin information)
- **Team identification** details
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/0a3f2d3f-179a-4ad7-916e-b77a13dca01d/ascreenshot_ded5970762d54528ae656421148116c4_text_export.jpeg)
### Step 3: View Team Financial Information
Similar to deleted keys, the Deleted Teams table shows financial information:
- **Spend at deletion** - Total spend accumulated when the team was deleted
- **Original budget** - The budget limit that was set for the team
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-17/5b24871f-b57e-404d-8fbe-a4b27cb2a6a0/ascreenshot_3121fbafbd6b4abf90993ce6c03c608d_text_export.jpeg)
## Use Cases
This feature is particularly useful for:
- **Financial Auditing** - Track spend and budgets for deleted entities
- **Compliance** - Maintain records of who deleted what and when
- **Cost Analysis** - Understand spending patterns before deletion
- **Accountability** - Identify which admin or user performed deletions
- **Historical Records** - Preserve financial data even after entity deletion
## Related Features
- [Audit Logs](./multiple_admins.md) - View comprehensive audit logs for all entity changes
- [UI Logs](./ui_logs.md) - View request logs and spend tracking

5
docs/my-website/docs/proxy/deploy.md
View file

@ -4,6 +4,10 @@ import Image from '@theme/IdealImage';
# Docker, Helm, Terraform
:::info No Limits on LiteLLM OSS
There are **no limits** on the number of users, keys, or teams you can create on LiteLLM OSS.
:::
You can find the Dockerfile to build litellm proxy [here](https://github.com/BerriAI/litellm/blob/main/Dockerfile)
> Note: Production requires at least 4 CPU cores and 8 GB RAM.
@ -196,6 +200,7 @@ Example `requirements.txt`
```shell
litellm[proxy]==1.57.3 # Specify the litellm version you want to use
litellm-enterprise
prometheus_client
langfuse
prisma

10
docs/my-website/docs/proxy/embedding.md
View file

@ -6,6 +6,16 @@ import TabItem from '@theme/TabItem';
See supported Embedding Providers & Models [here](https://docs.litellm.ai/docs/embedding/supported_embedding)
## Supported Input Formats
The `/v1/embeddings` endpoint follows the [OpenAI embeddings API specification](https://platform.openai.com/docs/api-reference/embeddings/create). The following input formats are supported:
| Format | Example |
|--------|---------|
| String | `"input": "Hello"` |
| Array of strings | `"input": ["Hello", "World"]` |
| Array of tokens (integers) | `"input": [1234, 5678, 9012]` |
| Array of token arrays | `"input": [[1234, 5678], [9012, 3456]]` |
## Quick start
Here's how to route between GPT-J embedding (sagemaker endpoint), Amazon Titan embedding (Bedrock) and Azure OpenAI embedding on the proxy server:

267
docs/my-website/docs/proxy/fallback_management.md Normal file
View file

@ -0,0 +1,267 @@
# [New] Fallback Management Endpoints
Dedicated endpoints for managing model fallbacks separately from the general configuration.
## Overview
These endpoints allow you to configure, retrieve, and delete fallback models without modifying the entire proxy configuration. This provides a cleaner and safer way to manage fallbacks compared to using the `/config/update` endpoint.
## Prerequisites
- Database storage must be enabled: Set `STORE_MODEL_IN_DB=True` in your environment
- Models must exist in the router before configuring fallbacks
## Endpoints
### POST /fallback
Create or update fallbacks for a specific model.
**Request Body:**
```json
{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general"
}
```
**Parameters:**
- `model` (string, required): The primary model name to configure fallbacks for
- `fallback_models` (array of strings, required): List of fallback model names in priority order
- `fallback_type` (string, optional): Type of fallback. Options:
- `"general"` (default): Standard fallbacks for any error
- `"context_window"`: Fallbacks for context window exceeded errors
- `"content_policy"`: Fallbacks for content policy violations
**Response:**
```json
{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general",
"message": "Fallback configuration created successfully"
}
```
**Example using cURL:**
```bash
curl -X POST "http://localhost:4000/fallback" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general"
}'
```
**Example using Python:**
```python
import requests
response = requests.post(
"http://localhost:4000/fallback",
headers={
"Authorization": "Bearer sk-1234",
"Content-Type": "application/json"
},
json={
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general"
}
)
print(response.json())
```
### GET /fallback/\{model\}
Get fallback configuration for a specific model.
**Parameters:**
- `model` (path parameter, required): The model name to get fallbacks for
- `fallback_type` (query parameter, optional): Type of fallback to retrieve (default: "general")
**Response:**
```json
{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general"
}
```
**Example using cURL:**
```bash
curl -X GET "http://localhost:4000/fallback/gpt-3.5-turbo?fallback_type=general" \
-H "Authorization: Bearer sk-1234"
```
**Example using Python:**
```python
import requests
response = requests.get(
"http://localhost:4000/fallback/gpt-3.5-turbo",
headers={"Authorization": "Bearer sk-1234"},
params={"fallback_type": "general"}
)
print(response.json())
```
### DELETE /fallback/\{model\}
Delete fallback configuration for a specific model.
**Parameters:**
- `model` (path parameter, required): The model name to delete fallbacks for
- `fallback_type` (query parameter, optional): Type of fallback to delete (default: "general")
**Response:**
```json
{
"model": "gpt-3.5-turbo",
"fallback_type": "general",
"message": "Fallback configuration deleted successfully"
}
```
**Example using cURL:**
```bash
curl -X DELETE "http://localhost:4000/fallback/gpt-3.5-turbo?fallback_type=general" \
-H "Authorization: Bearer sk-1234"
```
**Example using Python:**
```python
import requests
response = requests.delete(
"http://localhost:4000/fallback/gpt-3.5-turbo",
headers={"Authorization": "Bearer sk-1234"},
params={"fallback_type": "general"}
)
print(response.json())
```
### Test fallback
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "ping"
}
],
"mock_testing_fallbacks": true
}
'
```
## Validation
The endpoints perform the following validations:
1. **Model Existence**: Verifies that the primary model exists in the router
2. **Fallback Model Existence**: Ensures all fallback models exist in the router
3. **No Self-Fallback**: Prevents a model from being its own fallback
4. **No Duplicates**: Ensures no duplicate models in the fallback list
5. **Database Enabled**: Requires `STORE_MODEL_IN_DB=True` to be set
## Error Responses
### 400 Bad Request
```json
{
"detail": {
"error": "Invalid fallback models: ['non-existent-model']",
"available_models": ["gpt-3.5-turbo", "gpt-4", "claude-3-haiku"]
}
}
```
### 404 Not Found
```json
{
"detail": {
"error": "Model 'gpt-3.5-turbo' not found in router",
"available_models": ["gpt-4", "claude-3-haiku"]
}
}
```
### 500 Internal Server Error
```json
{
"detail": {
"error": "Router not initialized"
}
}
```
## Fallback Types Explained
### General Fallbacks
Used for any type of error that occurs during model invocation. This is the most common type of fallback.
**Use Case:** When a model is unavailable, rate-limited, or returns an error.
```json
{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4", "claude-3-haiku"],
"fallback_type": "general"
}
```
### Context Window Fallbacks
Specifically triggered when a context window exceeded error occurs.
**Use Case:** When the input is too long for the primary model, fallback to a model with a larger context window.
```json
{
"model": "gpt-3.5-turbo",
"fallback_models": ["gpt-4-32k", "claude-3-opus"],
"fallback_type": "context_window"
}
```
### Content Policy Fallbacks
Specifically triggered when content policy violations occur.
**Use Case:** When the primary model rejects content due to safety filters, fallback to a model with different content policies.
```json
{
"model": "gpt-4",
"fallback_models": ["claude-3-haiku"],
"fallback_type": "content_policy"
}
```
## Benefits Over /config/update
1. **Safety**: Only modifies fallback configuration, won't accidentally change other settings
2. **Simplicity**: Focused API with clear validation messages
3. **Granularity**: Manage fallbacks per model and per type
4. **Validation**: Comprehensive checks ensure configuration is valid before applying
5. **Clarity**: Clear error messages with available models listed
## Notes
- Fallbacks are triggered after the configured number of retries fails
- Fallbacks are attempted in the order specified in `fallback_models`
- The maximum number of fallbacks attempted is controlled by the router's `max_fallbacks` setting
- Changes take effect immediately and are persisted to the database

1
docs/my-website/docs/proxy/guardrails/aim_security.md
View file

@ -46,6 +46,7 @@ guardrails:
mode: [pre_call, post_call] # "During_call" is also available
api_key: os.environ/AIM_API_KEY
api_base: os.environ/AIM_API_BASE # Optional, use only when using a self-hosted Aim Outpost
ssl_verify: False # Optional, set to False to disable SSL verification or a string path to a custom CA bundle
```
Under the `api_key`, insert the API key you were issued. The key can be found in the guard's page.

283
docs/my-website/docs/proxy/guardrails/guardrail_policies.md Normal file
View file

@ -0,0 +1,283 @@
# [Beta] Guardrail Policies
Use policies to group guardrails and control which ones run for specific teams, keys, or models.
## Why use policies?
- Enable/disable specific guardrails for teams, keys, or models
- Group guardrails into a single policy
- Inherit from existing policies and override what you need
## Quick Start
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
# 1. Define your guardrails
guardrails:
- guardrail_name: pii_masking
litellm_params:
guardrail: presidio
mode: pre_call
- guardrail_name: prompt_injection
litellm_params:
guardrail: lakera
mode: pre_call
api_key: os.environ/LAKERA_API_KEY
# 2. Create a policy
policies:
my-policy:
guardrails:
add:
- pii_masking
- prompt_injection
# 3. Attach the policy
policy_attachments:
- policy: my-policy
scope: "*" # apply to all requests
```
Response headers show what ran:
```
x-litellm-applied-policies: my-policy
x-litellm-applied-guardrails: pii_masking,prompt_injection
```
## Add guardrails for a specific team
:::info
✨ Enterprise only feature for team/key-based policy attachments. [Get a free trial](https://www.litellm.ai/enterprise#trial)
:::
You have a global baseline, but want to add extra guardrails for a specific team.
```yaml showLineNumbers title="config.yaml"
policies:
global-baseline:
guardrails:
add:
- pii_masking
finance-team-policy:
inherit: global-baseline
guardrails:
add:
- strict_compliance_check
- audit_logger
policy_attachments:
- policy: global-baseline
scope: "*"
- policy: finance-team-policy
teams:
- finance # team alias from /team/new
```
Now the `finance` team gets `pii_masking` + `strict_compliance_check` + `audit_logger`, while everyone else just gets `pii_masking`.
## Remove guardrails for a specific team
:::info
✨ Enterprise only feature for team/key-based policy attachments. [Get a free trial](https://www.litellm.ai/enterprise#trial)
:::
You have guardrails running globally, but want to disable some for a specific team (e.g., internal testing).
```yaml showLineNumbers title="config.yaml"
policies:
global-baseline:
guardrails:
add:
- pii_masking
- prompt_injection
internal-team-policy:
inherit: global-baseline
guardrails:
remove:
- pii_masking # don't need PII masking for internal testing
policy_attachments:
- policy: global-baseline
scope: "*"
- policy: internal-team-policy
teams:
- internal-testing # team alias from /team/new
```
Now the `internal-testing` team only gets `prompt_injection`, while everyone else gets both guardrails.
## Inheritance
Start with a base policy and build on it:
```yaml showLineNumbers title="config.yaml"
policies:
base:
guardrails:
add:
- pii_masking
- toxicity_filter
strict:
inherit: base
guardrails:
add:
- prompt_injection
relaxed:
inherit: base
guardrails:
remove:
- toxicity_filter
```
What you get:
- `base` → `[pii_masking, toxicity_filter]`
- `strict` → `[pii_masking, toxicity_filter, prompt_injection]`
- `relaxed` → `[pii_masking]`
## Model Conditions
Run guardrails only for specific models:
```yaml showLineNumbers title="config.yaml"
policies:
gpt4-safety:
guardrails:
add:
- strict_content_filter
condition:
model: "gpt-4.*" # regex - matches gpt-4, gpt-4-turbo, gpt-4o
bedrock-compliance:
guardrails:
add:
- audit_logger
condition:
model: # exact match list
- bedrock/claude-3
- bedrock/claude-2
```
## Attachments
Policies don't do anything until you attach them. Attachments tell LiteLLM *where* to apply each policy.
**Global** - runs on every request:
```yaml showLineNumbers title="config.yaml"
policy_attachments:
- policy: default
scope: "*"
```
**Team-specific** (uses team alias from `/team/new`):
```yaml showLineNumbers title="config.yaml"
policy_attachments:
- policy: hipaa-compliance
teams:
- healthcare-team # team alias
- medical-research # team alias
```
**Key-specific** (uses key alias from `/key/generate`, wildcards supported):
```yaml showLineNumbers title="config.yaml"
policy_attachments:
- policy: internal-testing
keys:
- "dev-*" # key alias pattern
- "test-*" # key alias pattern
```
## Config Reference
### `policies`
```yaml
policies:
<policy-name>:
description: ...
inherit: ...
guardrails:
add: [...]
remove: [...]
condition:
model: ...
```
| Field | Type | Description |
|-------|------|-------------|
| `description` | `string` | Optional. What this policy does. |
| `inherit` | `string` | Optional. Parent policy to inherit guardrails from. |
| `guardrails.add` | `list[string]` | Guardrails to enable. |
| `guardrails.remove` | `list[string]` | Guardrails to disable (useful with inheritance). |
| `condition.model` | `string` or `list[string]` | Optional. Only apply when model matches. Supports regex. |
### `policy_attachments`
```yaml
policy_attachments:
- policy: ...
scope: ...
teams: [...]
keys: [...]
```
| Field | Type | Description |
|-------|------|-------------|
| `policy` | `string` | **Required.** Name of the policy to attach. |
| `scope` | `string` | Use `"*"` to apply globally. |
| `teams` | `list[string]` | Team aliases (from `/team/new`). |
| `keys` | `list[string]` | Key aliases (from `/key/generate`). Supports `*` wildcard. |
### Response Headers
| Header | Description |
|--------|-------------|
| `x-litellm-applied-policies` | Policies that matched this request |
| `x-litellm-applied-guardrails` | Guardrails that actually ran |
## How it works
Example config:
```yaml showLineNumbers title="config.yaml"
policies:
base:
guardrails:
add: [pii_masking]
finance-policy:
inherit: base
guardrails:
add: [audit_logger]
policy_attachments:
- policy: base
scope: "*"
- policy: finance-policy
teams: [finance]
```
```mermaid
flowchart TD
A["Request with team_alias='finance'"] --> B["Matches policies: base, finance-policy"]
B --> C["Resolves guardrails: pii_masking, audit_logger"]
```
1. Request comes in with `team_alias='finance'`
2. Matches `base` (via `scope: "*"`) and `finance-policy` (via `teams: [finance]`)
3. Resolves guardrails: `base` adds `pii_masking`, `finance-policy` inherits and adds `audit_logger`
4. Final guardrails: `pii_masking`, `audit_logger`

3
docs/my-website/docs/proxy/guardrails/onyx_security.md
View file

@ -128,6 +128,7 @@ guardrails:
mode: ["pre_call", "post_call", "during_call"] # Run at multiple stages
api_key: os.environ/ONYX_API_KEY
api_base: os.environ/ONYX_API_BASE
timeout: 10.0 # Optional, defaults to 10 seconds
```
### Required Parameters
@ -137,6 +138,7 @@ guardrails:
### Optional Parameters
- **`api_base`**: Onyx API base URL (defaults to `https://ai-guard.onyx.security`)
- **`timeout`**: Request timeout in seconds (defaults to `10.0`)
## Environment Variables
@ -145,4 +147,5 @@ You can set these environment variables instead of hardcoding values in your con
```shell
export ONYX_API_KEY="your-api-key-here"
export ONYX_API_BASE="https://ai-guard.onyx.security" # Optional
export ONYX_TIMEOUT=10 # Optional, timeout in seconds
```

28
docs/my-website/docs/proxy/guardrails/panw_prisma_airs.md
View file

@ -206,6 +206,7 @@ Expected successful response:
| `mode` | No | When to run the guardrail | `pre_call` |
| `fallback_on_error` | No | Action when PANW API is unavailable: `"block"` (fail-closed, default) or `"allow"` (fail-open). Config errors always block. | `block` |
| `timeout` | No | PANW API call timeout in seconds (1-60) | `10.0` |
| `violation_message_template` | No | Custom template for error message when request is blocked. Supports `{guardrail_name}`, `{category}`, `{action_type}`, `{default_message}` placeholders. | - |
### Regional Endpoints
@ -449,6 +450,33 @@ LiteLLM does not alter or configure your PANW security profile. To change what c
The guardrail is **fail-closed** by default - if the PANW API is unavailable, requests are blocked to ensure no unscanned content reaches your LLM. This provides maximum security.
:::
### Custom Violation Messages
You can customize the error message returned to the user when a request is blocked by configuring the `violation_message_template` parameter. This is useful for providing user-friendly feedback instead of technical details.
```yaml
guardrails:
- guardrail_name: "panw-custom-message"
litellm_params:
guardrail: panw_prisma_airs
api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
# Simple message
violation_message_template: "Your request was blocked by our AI Security Policy."
- guardrail_name: "panw-detailed-message"
litellm_params:
guardrail: panw_prisma_airs
api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
# Message with placeholders
violation_message_template: "{action_type} blocked due to {category} violation. Please contact support."
```
**Supported Placeholders:**
- `{guardrail_name}`: Name of the guardrail (e.g. "panw-custom-message")
- `{category}`: Violation category (e.g. "malicious", "injection", "dlp")
- `{action_type}`: "Prompt" or "Response"
- `{default_message}`: The original technical error message
### Fail-Open Configuration
By default, the PANW guardrail operates in **fail-closed** mode for maximum security. If the PANW API is unavailable (timeout, rate limit, network error), requests are blocked. You can configure **fail-open** mode for high-availability scenarios where service continuity is critical.

846
docs/my-website/docs/proxy/guardrails/pillar_security.md
View file

File diff suppressed because it is too large Load diff

24
docs/my-website/docs/proxy/guardrails/quick_start.md
View file

@ -59,6 +59,18 @@ guardrails:
presidio_score_thresholds: # minimum confidence scores for keeping detections
CREDIT_CARD: 0.8
EMAIL_ADDRESS: 0.6
# Example Pillar Security config via Generic Guardrail API
- guardrail_name: "pillar-security"
litellm_params:
guardrail: generic_guardrail_api
mode: [pre_call, post_call]
api_base: https://api.pillar.security/api/v1/integrations/litellm
api_key: os.environ/PILLAR_API_KEY
additional_provider_specific_params:
plr_mask: true
plr_evidence: true
plr_scanners: true
```
@ -191,8 +203,12 @@ Your response headers will include `x-litellm-applied-guardrails` with the guard
x-litellm-applied-guardrails: aporia-pre-guard
```
### Guardrail Policies
Need more control? Use [Guardrail Policies](./guardrail_policies.md) to:
- Group guardrails into reusable policies
- Enable/disable guardrails for specific teams, keys, or models
- Inherit from existing policies and override specific guardrails
## **Using Guardrails Client Side**
@ -389,14 +405,10 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
## **Proxy Admin Controls**
### ✨ Monitoring Guardrails
### Monitoring Guardrails
Monitor which guardrails were executed and whether they passed or failed. e.g. guardrail going rogue and failing requests we don't intend to fail
:::info
✨ This is an Enterprise only feature [Get a free trial](https://www.litellm.ai/enterprise#trial)
:::
#### Setup

150
docs/my-website/docs/proxy/keys_teams_router_settings.md Normal file
View file

@ -0,0 +1,150 @@
import Image from '@theme/IdealImage';
# UI - Router Settings for Keys and Teams
Configure router settings at the key and team level to achieve granular control over routing behavior, fallbacks, retries, and other router configurations. This enables you to customize routing behavior for specific keys or teams without affecting global settings.
## Overview
Router Settings for Keys and Teams allows you to configure router behavior at different levels of granularity. Previously, router settings could only be configured globally, applying the same routing strategy, fallbacks, timeouts, and retry policies to all requests across your entire proxy instance.
With key-level and team-level router settings, you can now:
- **Customize routing strategies** per key or team (e.g., use `least-busy` for high-priority keys, `latency-based-routing` for others)
- **Configure different fallback chains** for different keys or teams
- **Set key-specific or team-specific timeouts** and retry policies
- **Apply different reliability settings** (cooldowns, allowed failures) per key or team
- **Override global settings** when needed for specific use cases
<Image img={require('../../img/ui_granular_router_settings.png')} />
## Summary
Router settings follow a **hierarchical resolution order**: **Keys > Teams > Global**. When a request is made:
1. **Key-level settings** are checked first. If router settings are configured for the API key being used, those settings are applied.
2. **Team-level settings** are checked next. If the key belongs to a team and that team has router settings configured, those settings are used (unless key-level settings exist).
3. **Global settings** are used as the final fallback. If neither key nor team settings are found, the global router settings from your proxy configuration are applied.
This hierarchical approach ensures that the most specific settings take precedence, allowing you to fine-tune routing behavior for individual keys or teams while maintaining sensible defaults at the global level.
## How Router Settings Resolution Works
Router settings are resolved in the following priority order:
### Resolution Order: Key > Team > Global
1. **Key-level router settings** (highest priority)
- Applied when router settings are configured directly on an API key
- Takes precedence over all other settings
- Useful for individual key customization
2. **Team-level router settings** (medium priority)
- Applied when the API key belongs to a team with router settings configured
- Only used if no key-level settings exist
- Useful for applying consistent settings across multiple keys in a team
3. **Global router settings** (lowest priority)
- Applied from your proxy configuration file or database
- Used as the default when no key or team settings are found
- Previously, this was the only option available
## How to Configure Router Settings
### Configuring Router Settings for Keys
Follow these steps to configure router settings for an API key:
1. Navigate to [http://localhost:4000/ui/?login=success](http://localhost:4000/ui/?login=success)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/61889da3-32de-4ebf-9cf3-7dc1db2fc993/ascreenshot_2492cf6d916a4ab98197cc8336e3a371_text_export.jpeg)
2. Click "+ Create New Key" (or edit an existing key)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/61889da3-32de-4ebf-9cf3-7dc1db2fc993/ascreenshot_5a25380cf5044b4f93c146139d84403a_text_export.jpeg)
3. Click "Optional Settings"
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/e5eb5858-1cc1-4273-90bd-19ad139feebd/ascreenshot_33888989cfb9445bb83660f702ba32e0_text_export.jpeg)
4. Click "Router Settings"
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/d9eeca83-1f76-4fcf-bf61-d89edf3454d3/ascreenshot_825c7993f4b24949aee9b31d4a788d8a_text_export.jpeg)
5. Configure your desired router settings. For example, click "Fallbacks" to configure fallback models:
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/30ff647f-0254-4410-8311-660eef7ec0c4/ascreenshot_16966c8a0160473eb03e0f2c3b5c3afa_text_export.jpeg)
6. Click "Select a model to begin configuring fallbacks" and configure your fallback chain:
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/918f1b5b-c656-4864-98bd-d8c58924b6d9/ascreenshot_79ca6cd93be04033929f080e0c8d040a_text_export.jpeg)
### Configuring Router Settings for Teams
Follow these steps to configure router settings for a team:
1. Navigate to [http://localhost:4000/ui/?login=success](http://localhost:4000/ui/?login=success)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/60a33a8c-2e48-4788-a1a2-e5bcffa98cca/ascreenshot_9e255ba48f914c72ae57db7d3c1c7cd5_text_export.jpeg)
2. Click "Teams"
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/60a33a8c-2e48-4788-a1a2-e5bcffa98cca/ascreenshot_070934fa9c17453987f21f58117e673b_text_export.jpeg)
3. Click "+ Create New Team" (or edit an existing team)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/6f964ce2-f458-4719-a070-1af444ad92f5/ascreenshot_10f427f3106a4032a65d1046668880bd_text_export.jpeg)
4. Click "Router Settings"
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/a923c4ae-29f2-42b5-93ae-12f62d442691/ascreenshot_144520f2dd2f419dad79dffb1579ec04_text_export.jpeg)
5. Configure your desired router settings. For example, click "Fallbacks" to configure fallback models:
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/b062ecfa-bf5b-4c99-93a1-84b8b56fdb4c/ascreenshot_ea9acbc4e75448709b64a22addfb4157_text_export.jpeg)
6. Click "Select a model to begin configuring fallbacks" and configure your fallback chain:
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/67ca2655-4e82-4f93-be9a-7244ad22640f/ascreenshot_4fdbed826cd546d784e8738626be835d_text_export.jpeg)
## Use Cases
### Different Routing Strategies per Key
Configure different routing strategies for different use cases:
- **High-priority production keys**: Use `latency-based-routing` for optimal performance
- **Development keys**: Use `simple-shuffle` for simplicity
- **Cost-sensitive keys**: Use `cost-based-routing` to minimize expenses
### Team-Level Consistency
Apply consistent router settings across all keys in a team:
- Set team-wide fallback chains for reliability
- Configure team-specific timeout policies
- Apply uniform retry policies across team members
### Override Global Settings
Override global settings for specific scenarios:
- Production keys may need stricter timeout policies than development
- Certain teams may require different fallback models
- Individual keys may need custom retry policies for specific use cases
### Gradual Rollout
Test new router settings on specific keys or teams before applying globally:
- Configure new routing strategies on a test key first
- Validate fallback chains on a small team before global rollout
- A/B test different timeout values across different keys
## Related Features
- [Router Settings Reference](./config_settings.md#router_settings---reference) - Complete reference of all router settings
- [Load Balancing](./load_balancing.md) - Learn about routing strategies and load balancing
- [Reliability](./reliability.md) - Configure fallbacks, retries, and error handling
- [Keys](./keys.md) - Manage API keys and their settings
- [Teams](./teams.md) - Organize keys into teams

2
docs/my-website/docs/proxy/litellm_managed_files.md
View file

@ -11,7 +11,7 @@ import Image from '@theme/IdealImage';
This is a free LiteLLM Enterprise feature.
Available via the `litellm[proxy]` package or any `litellm` docker image.
Available via the `litellm` docker image. If you are using the pip package, you must install [`litellm-enterprise`](https://pypi.org/project/litellm-enterprise/).
:::

60
docs/my-website/docs/proxy/logging.md
View file

@ -982,6 +982,8 @@ OTEL_ENDPOINT="http:/0.0.0.0:4317"
OTEL_HEADERS="x-honeycomb-team=<your-api-key>" # Optional
```
> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`).
Add `otel` as a callback on your `litellm_config.yaml`
```shell
@ -1827,6 +1829,64 @@ This approach allows you to:
- Share callbacks across different environments
- Version control callback files in cloud storage
#### Step 2c - Mounting Custom Callbacks in Helm/Kubernetes (Alternative)
When deploying with Helm or Kubernetes, you can mount custom callback Python files alongside your `config.yaml` using `subPath` to avoid overwriting the config directory.
**The Problem:**
Mounting a volume to a directory (e.g., `/app/`) would normally hide all existing files in that directory, including your `config.yaml`.
**The Solution:**
Use `subPath` in your `volumeMounts` to mount individual files without overwriting the entire directory.
**Example - Helm values.yaml:**
```yaml
# values.yaml
volumes:
- name: callback-files
configMap:
name: litellm-callback-files
volumeMounts:
- name: callback-files
mountPath: /app/custom_callbacks.py # Mount to specific FILE path
subPath: custom_callbacks.py # Required to avoid overwriting directory
```
**Create the ConfigMap with your callback file:**
```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: litellm-callback-files
data:
custom_callbacks.py: |
from litellm.integrations.custom_logger import CustomLogger
class MyCustomHandler(CustomLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
print(f"Success! Model: {kwargs.get('model')}")
proxy_handler_instance = MyCustomHandler()
```
**Reference in your config.yaml:**
```yaml
litellm_settings:
callbacks: custom_callbacks.proxy_handler_instance
```
**How it works:**
1. The `subPath` parameter tells Kubernetes to mount only the specific file
2. This places `custom_callbacks.py` in `/app/` alongside your existing `config.yaml`
3. LiteLLM automatically finds the callback file in the same directory as the config
4. No files are overwritten or hidden
**Note:** You can mount multiple callback files by adding more `volumeMounts` entries, each with its own `subPath`.
#### Step 3 - Start proxy + test request
```shell

5
docs/my-website/docs/proxy/prod.md
View file

@ -277,8 +277,13 @@ Set the following environment variable(s):
```bash
SEPARATE_HEALTH_APP="1" # Default "0"
SEPARATE_HEALTH_PORT="8001" # Default "4001", Works only if `SEPARATE_HEALTH_APP` is "1"
SUPERVISORD_STOPWAITSECS="3600" # Optional: Upper bound timeout in seconds for graceful shutdown. Default: 3600 (1 hour). Only used when SEPARATE_HEALTH_APP=1.
```
**Graceful Shutdown:**
Previously, `stopwaitsecs` was not set, defaulting to 10 seconds and causing in-flight requests to fail. `SUPERVISORD_STOPWAITSECS` (default: 3600) provides an upper bound for graceful shutdown, allowing uvicorn to wait for all in-flight requests to complete.
<video controls width="100%" style={{ borderRadius: '8px', marginBottom: '1em' }}>
<source src="https://cdn.loom.com/sessions/thumbnails/b08be303331246b88fdc053940d03281-1718990992822.mp4" type="video/mp4" />
Your browser does not support the video tag.

15
docs/my-website/docs/proxy/prometheus.md
View file

@ -121,8 +121,8 @@ Use this to track overall LiteLLM Proxy usage.
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "exception_status", "exception_class", "route"` |
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route"` |
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "user_email", "exception_status", "exception_class", "route", "model_id"` |
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"` |
### Callback Logging Metrics
@ -130,7 +130,12 @@ Monitor failures while shipping logs to downstream callbacks like `s3_v3` cold s
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_callback_logging_failures_metric` | Total number of failed attempts to emit logs to a configured callback. Labels: `"callback_name"`. Use this to alert on callback delivery issues such as repeated failures when writing to `s3_v3`. |
| `litellm_callback_logging_failures_metric` | Total number of failed attempts to emit logs to a configured callback. Labels: `"callback_name"`. Use this to alert on callback delivery issues such as repeated failures when writing to `s3_v3`, `langfuse`, or `langfuse_otel` and other otel providers |
**Supported Callbacks:**
- `S3Logger` - S3 v2 cold storage failures
- `langfuse` - Langfuse logging failures
- `otel` - OpenTelemetry logging failures
## LLM Provider Metrics
@ -191,10 +196,10 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model" |
| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model", "model_id" |
| `litellm_overhead_latency_metric` | Latency overhead (seconds) added by LiteLLM processing - tracked for labels "model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias" |
| `litellm_llm_api_latency_metric` | Latency (seconds) for just the LLM API call - tracked for labels "model", "hashed_api_key", "api_key_alias", "team", "team_alias", "requested_model", "end_user", "user" |
| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias` [Note: only emitted for streaming requests] |
| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias`, `requested_model`, `end_user`, `user`, `model_id` [Note: only emitted for streaming requests] |
## Tracking `end_user` on Prometheus

12
docs/my-website/docs/proxy/spend_logs_deletion.md
View file

@ -30,6 +30,9 @@ general_settings:
# Optional: set how frequently cleanup should run - default is daily
maximum_spend_logs_retention_interval: "1d" # Run cleanup daily
# Optional: set exact time for cleanup (Cron syntax)
maximum_spend_logs_cleanup_cron: "0 4 * * *" # Run at 04:00 AM daily
litellm_settings:
cache: true
cache_params:
@ -51,6 +54,15 @@ How long logs should be kept before deletion. Supported formats:
How often the cleanup job should run. Uses the same format as above. If not set, cleanup will run every 24 hours if and only if `maximum_spend_logs_retention_period` is set.
#### `maximum_spend_logs_cleanup_cron` (optional)
Schedule the cleanup using standard cron syntax. This takes precedence over `maximum_spend_logs_retention_interval`.
Examples:
- `"0 4 * * *"` – Run at 04:00 AM daily
- `"0 0 * * 0"` – Run at midnight every Sunday
- `"*/30 * * * *"` – Run every 30 minutes
## How it works
### Step 1. Lock Acquisition (Optional with Redis)

Some files were not shown because too many files have changed in this diff Show more