scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

Merge upstream/main and resolve conflicts in model_prices_and_context_window.json

Browse source 
Kept both cache_read_input_token_cost from upstream and deprecation_date from this branch for xai/grok-3-mini and xai/grok-3-mini-beta models.
This commit is contained in:
Chesars 2026-02-04 23:39:33 -03:00
commit 62a99535f5
978 changed files with 70096 additions and 24821 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

222
.circleci/config.yml
View file

@ -715,8 +715,8 @@ jobs:
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_router_coverage.xml
mv .coverage litellm_router_coverage
mv coverage.xml litellm_router_unit_coverage.xml
mv .coverage litellm_router_unit_coverage
# Store test results
- store_test_results:
path: test-results
@ -724,8 +724,8 @@ jobs:
- persist_to_workspace:
root: .
paths:
- litellm_router_coverage.xml
- litellm_router_coverage
- litellm_router_unit_coverage.xml
- litellm_router_unit_coverage
litellm_security_tests:
machine:
image: ubuntu-2204:2023.10.1
@ -1255,7 +1255,15 @@ jobs:
ls
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
# Subdirectories with dedicated jobs (maintain this list as new jobs are added)
IGNORE_DIRS=(
"tests/llm_translation/realtime"
)
IGNORE_ARGS=""
for dir in "${IGNORE_DIRS[@]}"; do
IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
done
python -m pytest -vv tests/llm_translation $IGNORE_ARGS --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -1271,6 +1279,54 @@ jobs:
paths:
- llm_translation_coverage.xml
- llm_translation_coverage
realtime_translation_testing:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pytest-xdist==3.6.1"
pip install "pytest-timeout==2.2.0"
pip install "websockets"
# Run pytest and generate JUnit XML report
- run:
name: Run realtime tests
command: |
pwd
ls
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml realtime_translation_coverage.xml
mv .coverage realtime_translation_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- realtime_translation_coverage.xml
- realtime_translation_coverage
mcp_testing:
docker:
- image: cimg/python:3.11
@ -3407,6 +3463,110 @@ jobs:
- store_test_results:
path: test-results
proxy_e2e_anthropic_messages_tests:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.10
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.10 -y
conda activate myenv
python --version
- run:
name: Install Dependencies
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
pip install "pytest==7.3.1"
pip install "pytest-asyncio==0.21.1"
pip install "boto3==1.36.0"
pip install "httpx==0.27.0"
pip install "claude-agent-sdk"
pip install -r requirements.txt
- run:
name: Install dockerize
command: |
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container with test config
command: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME="us-east-1" \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
- run:
name: Start outputting logs
command: docker logs -f my-app
background: true
- run:
name: Wait for app to be ready
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Run Claude Agent SDK E2E Tests
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
pwd
ls
python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
# Store test results
- store_test_results:
path: test-results
upload-coverage:
docker:
- image: cimg/python:3.9
@ -3428,7 +3588,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage combine llm_translation_coverage realtime_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@ -3478,8 +3638,22 @@ jobs:
ls dist/
twine upload --verbose dist/*
else
echo "Version ${VERSION} of package is already published on PyPI. Skipping PyPI publish."
circleci step halt
echo "Version ${VERSION} of package is already published on PyPI."
# Check if corresponding Docker nightly image exists
NIGHTLY_TAG="v${VERSION}-nightly"
echo "Checking for Docker nightly image: litellm/litellm:${NIGHTLY_TAG}"
# Check Docker Hub for the nightly image
if curl -s "https://hub.docker.com/v2/repositories/litellm/litellm/tags/${NIGHTLY_TAG}" | grep -q "name"; then
echo "Docker nightly image ${NIGHTLY_TAG} exists. This release was already completed successfully."
echo "Skipping PyPI publish and continuing to ensure Docker images are up to date."
circleci step halt
else
echo "ERROR: PyPI package ${VERSION} exists but Docker nightly image ${NIGHTLY_TAG} does not exist!"
echo "This indicates an incomplete release. Please investigate."
exit 1
fi
fi
- run:
name: Trigger Github Action for new Docker Container + Trigger Load Testing
@ -3488,11 +3662,21 @@ jobs:
python3 -m pip install toml
VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
echo "LiteLLM Version ${VERSION}"
# Determine which branch to use for Docker build
if [[ "$CIRCLE_BRANCH" =~ ^litellm_release_day_.* ]]; then
BUILD_BRANCH="$CIRCLE_BRANCH"
echo "Using release branch: $BUILD_BRANCH"
else
BUILD_BRANCH="main"
echo "Using default branch: $BUILD_BRANCH"
fi
curl -X POST \
-H "Accept: application/vnd.github.v3+json" \
-H "Authorization: Bearer $GITHUB_TOKEN" \
"https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \
-d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
-d "{\"ref\":\"${BUILD_BRANCH}\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
echo "triggering load testing server for version ${VERSION} and commit ${CIRCLE_SHA1}"
curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}&release_type=nightly"
@ -3626,6 +3810,9 @@ jobs:
cd ui/litellm-dashboard
# Remove node_modules and package-lock to ensure clean install (fixes dependency resolution issues)
rm -rf node_modules package-lock.json
# Install dependencies first
npm install
@ -4051,12 +4238,26 @@ workflows:
only:
- main
- /litellm_.*/
- proxy_e2e_anthropic_messages_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- llm_translation_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- realtime_translation_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- mcp_testing:
filters:
branches:
@ -4168,6 +4369,7 @@ workflows:
- upload-coverage:
requires:
- llm_translation_testing
- realtime_translation_testing
- mcp_testing
- google_generate_content_endpoint_testing
- guardrails_testing
@ -4235,6 +4437,7 @@ workflows:
branches:
only:
- main
- /litellm_release_day_.*/
- publish_to_pypi:
requires:
- mypy_linting
@ -4244,6 +4447,7 @@ workflows:
- e2e_openai_endpoints
- test_bad_database_url
- llm_translation_testing
- realtime_translation_testing
- mcp_testing
- google_generate_content_endpoint_testing
- llm_responses_api_testing

3
.circleci/requirements.txt
View file

@ -16,4 +16,5 @@ uvloop==0.21.0
mcp==1.25.0 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
fastuuid==0.12.0
responses==0.25.7 # for proxy client tests
responses==0.25.7 # for proxy client tests
pytest-retry==1.6.3 # for automatic test retries

2
.github/workflows/test-linting.yml vendored
View file

@ -73,4 +73,4 @@ jobs:
- name: Check import safety
run: |
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)

2
.github/workflows/test-litellm.yml vendored
View file

@ -34,7 +34,7 @@ jobs:
poetry run pip install "google-genai==1.22.0"
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.18"
poetry run pip install "python-multipart==0.0.22"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |

15
.github/workflows/test-model-map.yaml vendored Normal file
View file

@ -0,0 +1,15 @@
name: Validate model_prices_and_context_window.json
on:
pull_request:
branches: [ main ]
jobs:
validate-model-prices-json:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Validate model_prices_and_context_window.json
run: |
jq empty model_prices_and_context_window.json

9
.gitignore vendored
View file

@ -60,10 +60,6 @@ litellm/proxy/_super_secret_config.yaml
litellm/proxy/myenv/bin/activate
litellm/proxy/myenv/bin/Activate.ps1
myenv/*
litellm/proxy/_experimental/out/_next/
litellm/proxy/_experimental/out/404/index.html
litellm/proxy/_experimental/out/model_hub/index.html
litellm/proxy/_experimental/out/onboarding/index.html
litellm/tests/log.txt
litellm/tests/langfuse.log
litellm/tests/langfuse.log
@ -76,9 +72,6 @@ tests/local_testing/log.txt
litellm/proxy/_new_new_secret_config.yaml
litellm/proxy/custom_guardrail.py
.mypy_cache/*
litellm/proxy/_experimental/out/404.html
litellm/proxy/_experimental/out/404.html
litellm/proxy/_experimental/out/model_hub.html
.mypy_cache/*
litellm/proxy/application.log
tests/llm_translation/vertex_test_account.json
@ -100,9 +93,9 @@ litellm_config.yaml
litellm/proxy/to_delete_loadtest_work/*
update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
litellm/proxy/_experimental/out/guardrails/index.html
scripts/test_vertex_ai_search.py
LAZY_LOADING_IMPROVEMENTS.md
STABILIZATION_TODO.md
**/test-results
**/playwright-report
**/*.storageState.json

12
.trivyignore Normal file
View file

@ -0,0 +1,12 @@
# LiteLLM Trivy Ignore File
# CVEs listed here are temporarily allowlisted pending fixes
# Next.js vulnerabilities in UI dashboard (next@14.2.35)
# Allowlisted: 2026-01-31, 7-day fix timeline
# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+
# HIGH: DoS via request deserialization
GHSA-h25m-26qc-wcjf
# MEDIUM: Image Optimizer DoS
CVE-2025-59471

8
AGENTS.md
View file

@ -51,12 +51,14 @@ LiteLLM is a unified interface for 100+ LLMs that:
### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
1. **Use Common Components as much as possible**:
1. **Tremor is DEPRECATED, do not use Tremor components in new features/changes**
- The only exception is the Tremor Table component and its required Tremor Table sub components.
2. **Use Common Components as much as possible**:
- These are usually defined in the `common_components` directory
- Use these components as much as possible and avoid building new components unless needed
- Tremor components are deprecated; prefer using Ant Design (AntD) as much as possible
2. **Testing**:
3. **Testing**:
- The codebase uses **Vitest** and **React Testing Library**
- **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
- **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)

9
Dockerfile
View file

@ -46,8 +46,9 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
# Ensure runtime stage runs as root
USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@latest
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -69,8 +70,8 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Generate prisma client
RUN prisma generate
# Generate prisma client using the correct schema
RUN prisma generate --schema=./litellm/proxy/schema.prisma
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh

1
README.md
View file

@ -267,6 +267,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
<td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
<td><h2>Netflix</h2></td>
<td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
</tr>
</table>

6
ci_cd/security_scans.sh
View file

@ -81,10 +81,10 @@ run_trivy_scans() {
echo "Running Trivy scans..."
echo "Scanning LiteLLM Docs..."
trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
echo "Scanning LiteLLM UI..."
trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
echo "Trivy scans completed successfully"
}
@ -137,6 +137,7 @@ run_grype_scans() {
"CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
"CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
"GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
"GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
"GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
"CVE-2025-59465" # We do not use Node in application runtime, only used for building Admin UI
@ -153,6 +154,7 @@ run_grype_scans() {
"CVE-2025-15367" # No fix available yet
"CVE-2025-12781" # No fix available yet
"CVE-2025-11468" # No fix available yet
"CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
)
# Build JSON array of allowlisted CVE IDs for jq

144
cookbook/anthropic_agent_sdk/README.md Normal file
View file

@ -0,0 +1,144 @@
# Claude Agent SDK with LiteLLM Gateway
A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK.
## Quick Start
### 1. Install dependencies
```bash
pip install anthropic claude-agent-sdk litellm
```
### 2. Start LiteLLM proxy
```bash
# Simple start with Claude
litellm --model claude-sonnet-4-20250514
# Or with a config file
litellm --config config.yaml
```
### 3. Run the chat
**Basic Agent (no MCP):**
```bash
python main.py
```
**Agent with MCP (DeepWiki2 for research):**
```bash
python agent_with_mcp.py
```
If MCP connection fails, you can disable it:
```bash
USE_MCP=false python agent_with_mcp.py
```
That's it! You can now chat with the agent in your terminal.
### Chat Commands
While chatting, you can use these commands:
- `models` - List all available models (fetched from your LiteLLM proxy)
- `model` - Switch to a different model
- `clear` - Start a new conversation
- `quit` or `exit` - End the chat
The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured.
## Configuration
Set these environment variables if needed:
```bash
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
export LITELLM_MODEL="bedrock-claude-sonnet-4.5"
```
Or just use the defaults - it'll connect to `http://localhost:4000` by default.
## Files
- `main.py` - Basic interactive agent without MCP
- `agent_with_mcp.py` - Agent with MCP server integration (DeepWiki2)
- `common.py` - Shared utilities and functions
- `config.example.yaml` - Example LiteLLM configuration
- `requirements.txt` - Python dependencies
## Example Config File
If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`):
```yaml
model_list:
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
```
Then start LiteLLM with: `litellm --config config.yaml`
## How It Works
The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
```python
# Point to LiteLLM gateway (not Anthropic)
os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key
# Use any model configured in LiteLLM
options = ClaudeAgentOptions(
model="bedrock-claude-sonnet-4", # or gpt-4, or anything else
system_prompt="You are a helpful assistant.",
max_turns=50,
)
```
Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically.
## Why Use This?
- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc.
- **Cost tracking**: LiteLLM tracks spending across all your agent conversations
- **Rate limiting**: Set budgets and limits on your agent usage
- **Load balancing**: Distribute requests across multiple API keys or regions
- **Fallbacks**: Automatically retry with a different model if one fails
## Troubleshooting
**Connection errors?**
- Make sure LiteLLM is running: `litellm --model your-model`
- Check the URL is correct (default: `http://localhost:4000`)
**Authentication errors?**
- Verify your LiteLLM API key is correct
- Make sure the model is configured in your LiteLLM setup
**Model not found?**
- Check the model name matches what's in your LiteLLM config
- Run `litellm --model your-model` to test it works
**Agent with MCP stuck or failing?**
- The MCP server might not be available at `http://localhost:4000/mcp/deepwiki2`
- Try disabling MCP: `USE_MCP=false python agent_with_mcp.py`
- Or use the basic agent: `python main.py`
## Learn More
- [LiteLLM Docs](https://docs.litellm.ai/)
- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk)
- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start)

140
cookbook/anthropic_agent_sdk/agent_with_mcp.py Normal file
View file

@ -0,0 +1,140 @@
"""
Interactive Claude Agent SDK CLI with MCP Support
This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy,
with MCP (Model Context Protocol) server integration for enhanced capabilities.
"""
import asyncio
import os
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
from common import (
Config,
fetch_available_models,
setup_litellm_env,
print_header,
handle_model_list,
handle_model_switch,
stream_response,
)
async def interactive_chat_with_mcp():
"""
Interactive CLI chat with the agent and MCP server
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
current_model = config.LITELLM_MODEL
# MCP server configuration
mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
if not use_mcp:
print("⚠️ MCP disabled via USE_MCP=false")
print_header(litellm_base_url, current_model, has_mcp=use_mcp)
while True:
# Configure agent options
if use_mcp:
try:
# Try with MCP server (HTTP transport)
# Using McpHttpServerConfig format from Agent SDK
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant with access to DeepWiki for research. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
mcp_servers={
"deepwiki2": {
"type": "http",
"url": mcp_server_url,
"headers": {
"Authorization": f"Bearer {config.LITELLM_API_KEY}"
}
}
},
)
except Exception as e:
print(f"⚠️ Warning: Could not configure MCP server: {e}")
print("Continuing without MCP...\n")
use_mcp = False
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
else:
# Without MCP
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
# Create agent client
try:
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
user_input = input("\n👤 You: ").strip()
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)
except Exception as e:
print(f"\n❌ Error creating agent client: {e}")
print("This might be an MCP configuration issue. Try running without MCP:")
print(" USE_MCP=false python agent_with_mcp.py")
print("\nOr use the basic agent:")
print(" python main.py")
return
def main():
"""Run interactive chat with MCP"""
try:
asyncio.run(interactive_chat_with_mcp())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
if __name__ == "__main__":
main()

160
cookbook/anthropic_agent_sdk/common.py Normal file
View file

@ -0,0 +1,160 @@
"""
Common utilities for Claude Agent SDK examples
"""
import os
import httpx
class Config:
"""Configuration for LiteLLM Gateway connection"""
# LiteLLM proxy URL (default to local instance)
LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
# LiteLLM API key (master key or virtual key)
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
# Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
"""
Fetch available models from LiteLLM proxy /models endpoint
"""
try:
async with httpx.AsyncClient() as client:
response = await client.get(
f"{base_url}/models",
headers={"Authorization": f"Bearer {api_key}"},
timeout=10.0
)
response.raise_for_status()
data = response.json()
return [model["id"] for model in data.get("data", [])]
except Exception as e:
print(f"⚠️ Warning: Could not fetch models from proxy: {e}")
print("Using default model list...")
# Fallback to default models
return [
"bedrock-claude-sonnet-3.5",
"bedrock-claude-sonnet-4",
"bedrock-claude-sonnet-4.5",
"bedrock-claude-opus-4.5",
"bedrock-nova-premier",
]
def setup_litellm_env(config: Config):
"""
Configure environment variables to point Agent SDK to LiteLLM
"""
litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
return litellm_base_url
def print_header(base_url: str, current_model: str, has_mcp: bool = False):
"""
Print the chat header
"""
mcp_indicator = " + MCP" if has_mcp else ""
print("=" * 70)
print(f"🤖 Claude Agent SDK with LiteLLM Gateway{mcp_indicator} - Interactive Chat")
print("=" * 70)
print(f"🚀 Connected to: {base_url}")
print(f"📦 Current model: {current_model}")
if has_mcp:
print("🔌 MCP: deepwiki2 enabled")
print("\nType your messages below. Commands:")
print(" - 'quit' or 'exit' to end the conversation")
print(" - 'clear' to start a new conversation")
print(" - 'model' to switch models")
print(" - 'models' to list available models")
print("=" * 70)
print()
def handle_model_list(available_models: list[str], current_model: str):
"""
Display available models
"""
print("\n📋 Available models:")
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
"""
Handle model switching
Returns:
tuple: (new_model, should_restart_conversation)
"""
print("\n📋 Select a model:")
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
try:
choice = input("\nEnter number (or press Enter to cancel): ").strip()
if choice:
idx = int(choice) - 1
if 0 <= idx < len(available_models):
new_model = available_models[idx]
print(f"\n✅ Switched to: {new_model}")
print("🔄 Starting new conversation with new model...\n")
return new_model, True
else:
print("❌ Invalid choice")
except (ValueError, IndexError):
print("❌ Invalid input")
return current_model, False
async def stream_response(client, user_input: str):
"""
Stream response from the agent
"""
print("\n🤖 Assistant: ", end='', flush=True)
try:
await client.query(user_input)
# Show loading indicator
print("⏳ thinking...", end='', flush=True)
# Stream the response
first_chunk = True
async for msg in client.receive_response():
# Clear loading indicator on first message
if first_chunk:
print("\r🤖 Assistant: ", end='', flush=True)
first_chunk = False
# Handle different message types
if hasattr(msg, 'type'):
if msg.type == 'content_block_delta':
# Streaming text delta
if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
print(msg.delta.text, end='', flush=True)
elif msg.type == 'content_block_start':
# Start of content block
if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
print(msg.content_block.text, end='', flush=True)
# Fallback to original content handling
if hasattr(msg, 'content'):
for content_block in msg.content:
if hasattr(content_block, 'text'):
print(content_block.text, end='', flush=True)
print() # New line after response
except Exception as e:
print(f"\r\n❌ Error: {e}")
print("Please check your LiteLLM gateway is running and configured correctly.")

25
cookbook/anthropic_agent_sdk/config.example.yaml Normal file
View file

@ -0,0 +1,25 @@
model_list:
- model_name: bedrock-claude-sonnet-3.5
litellm_params:
model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-opus-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-nova-premier
litellm_params:
model: "bedrock/amazon.nova-premier-v1:0"
aws_region_name: "us-east-1"

95
cookbook/anthropic_agent_sdk/main.py Normal file
View file

@ -0,0 +1,95 @@
"""
Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway
This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy.
LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.)
through the Claude Agent SDK by pointing it to the LiteLLM gateway.
"""
import asyncio
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
from common import (
Config,
fetch_available_models,
setup_litellm_env,
print_header,
handle_model_list,
handle_model_switch,
stream_response,
)
async def interactive_chat():
"""
Interactive CLI chat with the agent
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
current_model = config.LITELLM_MODEL
print_header(litellm_base_url, current_model)
while True:
# Configure agent options for each conversation
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
# Create agent client
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
user_input = input("\n👤 You: ").strip()
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)
def main():
"""Run interactive chat"""
try:
asyncio.run(interactive_chat())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
if __name__ == "__main__":
main()

2
cookbook/anthropic_agent_sdk/requirements.txt Normal file
View file

@ -0,0 +1,2 @@
claude-agent-sdk
httpx>=0.27.0

114
cookbook/livekit_agent_sdk/README.md Normal file
View file

@ -0,0 +1,114 @@
# LiveKit Voice Agent with LiteLLM Gateway
Simple example showing how to use LiveKit's xAI realtime plugin with LiteLLM as a proxy. This lets you switch between xAI, OpenAI, and Azure realtime APIs without changing your code.
## Quick Start
### 1. Install dependencies
```bash
pip install livekit-agents[xai] websockets
```
### 2. Start LiteLLM proxy
```bash
# With xAI
export XAI_API_KEY="your-xai-key"
litellm --config config.yaml --port 4000
```
### 3. Run the voice agent
```bash
python main.py
```
Type your message and get a voice response from Grok!
## Configuration
Set these environment variables if needed:
```bash
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
export LITELLM_MODEL="grok-voice-agent"
```
Or use the defaults - connects to `http://localhost:4000` by default.
## Example Config File
Create a `config.yaml` with your realtime models:
```yaml
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-2-vision-1212
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
- model_name: openai-voice-agent
litellm_params:
model: gpt-4o-realtime-preview
api_key: os.environ/OPENAI_API_KEY
model_info:
mode: realtime
general_settings:
master_key: sk-1234
```
Then start: `litellm --config config.yaml --port 4000`
## How It Works
LiveKit's xAI plugin connects through LiteLLM proxy by setting `base_url`:
```python
from livekit.plugins import xai
model = xai.realtime.RealtimeModel(
voice="ara",
api_key="sk-1234", # LiteLLM proxy key
base_url="http://localhost:4000", # Point to LiteLLM
)
```
## Switching Providers
Just change the model in your config - no code changes needed:
**xAI Grok:**
```yaml
model: xai/grok-2-vision-1212
```
**OpenAI:**
```yaml
model: gpt-4o-realtime-preview
```
**Azure OpenAI:**
```yaml
model: azure/gpt-4o-realtime-preview
api_base: https://your-endpoint.openai.azure.com/
```
## Why Use LiteLLM?
- ✅ **Switch providers** without changing agent code
- ✅ **Cost tracking** across all voice sessions
- ✅ **Rate limiting** and budgets
- ✅ **Load balancing** across multiple API keys
- ✅ **Fallbacks** to backup models
## Learn More
- [LiveKit xAI Realtime Tutorial](/docs/tutorials/livekit_xai_realtime)
- [xAI Realtime Docs](/docs/providers/xai_realtime)
- [LiveKit Agents Documentation](https://docs.livekit.io/agents/)
- [LiteLLM Realtime API](/docs/realtime)

21
cookbook/livekit_agent_sdk/config.example.yaml Normal file
View file

@ -0,0 +1,21 @@
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-2-vision-1212
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
- model_name: openai-voice-agent
litellm_params:
model: gpt-4o-realtime-preview
api_key: os.environ/OPENAI_API_KEY
model_info:
mode: realtime
litellm_settings:
drop_params: True
telemetry: False
general_settings:
master_key: sk-1234 # Change this to a secure key

112
cookbook/livekit_agent_sdk/main.py Normal file
View file

@ -0,0 +1,112 @@
"""
Simple xAI Voice Agent using LiveKit SDK with LiteLLM Gateway
This example shows how to use LiveKit's xAI realtime plugin through LiteLLM proxy.
LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI,
and Azure realtime APIs without changing your agent code.
"""
import asyncio
import json
import os
import websockets
# Configuration
PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
MODEL = os.getenv("LITELLM_MODEL", "grok-voice-agent")
async def run_voice_agent():
"""
Simple voice agent that:
1. Connects to xAI realtime API through LiteLLM proxy
2. Sends a user message
3. Streams back the response
"""
url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}"
headers = {"Authorization": f"Bearer {API_KEY}"}
print(f"🎙️ Connecting to voice agent...")
print(f" Model: {MODEL}")
print(f" Proxy: {PROXY_URL}")
print()
async with websockets.connect(url, additional_headers=headers) as ws:
# Receive initial connection event
initial = json.loads(await ws.recv())
print(f"✅ Connected! Event: {initial['type']}\n")
# Get user input
user_message = input("💬 Your message: ").strip()
if not user_message:
user_message = "Tell me a fun fact about AI!"
print(f"\n🤖 Sending to {MODEL}...\n")
# Send user message
await ws.send(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": user_message}]
}
}))
# Request response
await ws.send(json.dumps({
"type": "response.create",
"response": {"modalities": ["text", "audio"]}
}))
# Stream response
print("🎤 Response: ", end='', flush=True)
transcript = []
try:
while True:
msg = await asyncio.wait_for(ws.recv(), timeout=15.0)
event = json.loads(msg)
# Capture transcript deltas
if event['type'] == 'response.output_audio_transcript.delta':
delta = event.get('delta', '')
if delta:
print(delta, end='', flush=True)
transcript.append(delta)
# Done when response completes
elif event['type'] == 'response.done':
break
except asyncio.TimeoutError:
pass
print("\n")
if transcript:
print(f"✅ Complete response: {''.join(transcript)}")
await ws.close()
def main():
"""Run the voice agent"""
print("=" * 70)
print("LiveKit xAI Voice Agent via LiteLLM Proxy")
print("=" * 70)
print()
try:
asyncio.run(run_voice_agent())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
except Exception as e:
print(f"\n❌ Error: {e}")
print("\nMake sure LiteLLM proxy is running:")
print(f" litellm --config config.yaml --port 4000")
if __name__ == "__main__":
main()

2
cookbook/livekit_agent_sdk/requirements.txt Normal file
View file

@ -0,0 +1,2 @@
livekit-agents[xai]>=1.3.12
websockets>=15.0.1

284
cookbook/nova_sonic_realtime.py Normal file
View file

@ -0,0 +1,284 @@
"""
Client script to test Nova Sonic realtime API through LiteLLM proxy.
This script connects to LiteLLM proxy's realtime endpoint and enables
speech-to-speech conversation with Bedrock Nova Sonic.
Prerequisites:
- LiteLLM proxy running with Bedrock configured
- pyaudio installed: pip install pyaudio
- websockets installed: pip install websockets
Usage:
python nova_sonic_realtime.py
"""
import asyncio
import base64
import json
import pyaudio
import websockets
from typing import Optional
# Audio configuration (matching Nova Sonic requirements)
INPUT_SAMPLE_RATE = 16000 # Nova Sonic expects 16kHz input
OUTPUT_SAMPLE_RATE = 24000 # Nova Sonic outputs 24kHz
CHANNELS = 1
FORMAT = pyaudio.paInt16
CHUNK_SIZE = 1024
# LiteLLM proxy configuration
LITELLM_PROXY_URL = "ws://localhost:4000/v1/realtime?model=bedrock-sonic"
LITELLM_API_KEY = "sk-12345" # Your LiteLLM API key
class RealtimeClient:
"""Client for LiteLLM realtime API with audio support."""
def __init__(self, url: str, api_key: str):
self.url = url
self.api_key = api_key
self.ws: Optional[websockets.WebSocketClientProtocol] = None
self.is_active = False
self.audio_queue = asyncio.Queue()
self.pyaudio = pyaudio.PyAudio()
self.input_stream = None
self.output_stream = None
async def connect(self):
"""Connect to LiteLLM proxy realtime endpoint."""
print(f"Connecting to {self.url}...")
headers = {}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
self.ws = await websockets.connect(
self.url,
additional_headers=headers,
max_size=10 * 1024 * 1024, # 10MB max message size
)
self.is_active = True
print("✓ Connected to LiteLLM proxy")
async def send_session_update(self):
"""Send session configuration."""
session_update = {
"type": "session.update",
"session": {
"instructions": "You are a friendly assistant. Keep your responses short and conversational.",
"voice": "matthew",
"temperature": 0.8,
"max_response_output_tokens": 1024,
"modalities": ["text", "audio"],
"input_audio_format": "pcm16",
"output_audio_format": "pcm16",
"turn_detection": {
"type": "server_vad",
"threshold": 0.5,
"prefix_padding_ms": 300,
"silence_duration_ms": 500,
},
},
}
await self.ws.send(json.dumps(session_update))
print("✓ Session configuration sent")
async def receive_messages(self):
"""Receive and process messages from the server."""
try:
async for message in self.ws:
if not self.is_active:
break
try:
data = json.loads(message)
event_type = data.get("type")
if event_type == "session.created":
print(f"✓ Session created: {data.get('session', {}).get('id')}")
elif event_type == "response.created":
print("🤖 Assistant is responding...")
elif event_type == "response.text.delta":
# Print text transcription
delta = data.get("delta", "")
print(delta, end="", flush=True)
elif event_type == "response.audio.delta":
# Queue audio for playback
audio_b64 = data.get("delta", "")
if audio_b64:
audio_bytes = base64.b64decode(audio_b64)
await self.audio_queue.put(audio_bytes)
elif event_type == "response.text.done":
print() # New line after text
elif event_type == "response.done":
print("✓ Response complete")
elif event_type == "error":
print(f"❌ Error: {data.get('error', {})}")
else:
# Debug: print other event types
print(f"[{event_type}]", end=" ")
except json.JSONDecodeError:
print(f"Failed to parse message: {message[:100]}")
except websockets.exceptions.ConnectionClosed:
print("\n✗ Connection closed")
except Exception as e:
print(f"\n✗ Error receiving messages: {e}")
finally:
self.is_active = False
async def send_audio_chunk(self, audio_bytes: bytes):
"""Send audio chunk to server."""
if not self.is_active or not self.ws:
return
audio_b64 = base64.b64encode(audio_bytes).decode("utf-8")
message = {
"type": "input_audio_buffer.append",
"audio": audio_b64,
}
await self.ws.send(json.dumps(message))
async def commit_audio_buffer(self):
"""Commit the audio buffer to trigger processing."""
if not self.is_active or not self.ws:
return
message = {"type": "input_audio_buffer.commit"}
await self.ws.send(json.dumps(message))
async def capture_audio(self):
"""Capture audio from microphone and send to server."""
print("\n🎤 Starting audio capture...")
print("Speak into your microphone. Press Ctrl+C to stop.\n")
self.input_stream = self.pyaudio.open(
format=FORMAT,
channels=CHANNELS,
rate=INPUT_SAMPLE_RATE,
input=True,
frames_per_buffer=CHUNK_SIZE,
)
try:
while self.is_active:
audio_data = self.input_stream.read(CHUNK_SIZE, exception_on_overflow=False)
await self.send_audio_chunk(audio_data)
await asyncio.sleep(0.01) # Small delay to prevent overwhelming
except Exception as e:
print(f"Error capturing audio: {e}")
finally:
if self.input_stream:
self.input_stream.stop_stream()
self.input_stream.close()
async def play_audio(self):
"""Play audio responses from the server."""
print("🔊 Starting audio playback...")
self.output_stream = self.pyaudio.open(
format=FORMAT,
channels=CHANNELS,
rate=OUTPUT_SAMPLE_RATE,
output=True,
frames_per_buffer=CHUNK_SIZE,
)
try:
while self.is_active:
try:
audio_data = await asyncio.wait_for(
self.audio_queue.get(), timeout=0.1
)
if audio_data:
self.output_stream.write(audio_data)
except asyncio.TimeoutError:
continue
except Exception as e:
print(f"Error playing audio: {e}")
finally:
if self.output_stream:
self.output_stream.stop_stream()
self.output_stream.close()
async def close(self):
"""Close the connection and cleanup."""
self.is_active = False
if self.ws:
await self.ws.close()
if self.input_stream:
self.input_stream.stop_stream()
self.input_stream.close()
if self.output_stream:
self.output_stream.stop_stream()
self.output_stream.close()
self.pyaudio.terminate()
print("\n✓ Connection closed")
async def main():
"""Main function to run the realtime client."""
print("=" * 80)
print("Bedrock Nova Sonic Realtime Client")
print("=" * 80)
print()
client = RealtimeClient(LITELLM_PROXY_URL, LITELLM_API_KEY)
try:
# Connect to server
await client.connect()
# Send session configuration
await client.send_session_update()
# Wait a moment for session to be established
await asyncio.sleep(0.5)
# Start tasks
receive_task = asyncio.create_task(client.receive_messages())
capture_task = asyncio.create_task(client.capture_audio())
playback_task = asyncio.create_task(client.play_audio())
# Wait for user to interrupt
await asyncio.gather(
receive_task,
capture_task,
playback_task,
return_exceptions=True,
)
except KeyboardInterrupt:
print("\n\n⚠ Interrupted by user")
except Exception as e:
print(f"\n❌ Error: {e}")
import traceback
traceback.print_exc()
finally:
await client.close()
if __name__ == "__main__":
print("\nMake sure:")
print("1. LiteLLM proxy is running on port 4000")
print("2. Bedrock is configured in proxy_server_config.yaml")
print("3. AWS credentials are set")
print()
try:
asyncio.run(main())
except KeyboardInterrupt:
print("\n\nGoodbye!")

4
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -38,6 +38,10 @@ spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- with .Values.extraInitContainers }}
initContainers:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: {{ include "litellm.name" . }}
securityContext:

4
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -35,6 +35,10 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.migrationJob.extraInitContainers }}
initContainers:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: prisma-migrations
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}"

1
deploy/charts/litellm-helm/values.yaml
View file

@ -281,6 +281,7 @@ migrationJob:
# cpu: 100m
# memory: 100Mi
extraContainers: []
extraInitContainers: []
# Hook configuration
hooks:

3
docker/Dockerfile.custom_ui
View file

@ -5,7 +5,8 @@ FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
WORKDIR /app
# Install Node.js and npm (adjust version as needed)
RUN apt-get update && apt-get install -y nodejs npm
RUN apt-get update && apt-get install -y nodejs npm && \
npm install -g npm@latest tar@latest
# Copy the UI source into the container
COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard

3
docker/Dockerfile.database
View file

@ -49,7 +49,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@latest
WORKDIR /app
# Copy the current directory contents into the container at /app

3
docker/Dockerfile.dev
View file

@ -61,7 +61,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libatomic1 \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@latest tar@latest
WORKDIR /app

9
docker/Dockerfile.non_root
View file

@ -104,7 +104,8 @@ RUN for i in 1 2 3; do \
done \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done
done \
&& npm install -g npm@latest tar@latest
# Copy artifacts from builder
COPY --from=builder /app/requirements.txt /app/requirements.txt
@ -170,12 +171,14 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+rX $PRISMA_PATH && \
chmod -R g+rX /app/.cache && \
mkdir -p /tmp/.npm /nonexistent /.npm && \
prisma generate
mkdir -p /tmp/.npm /nonexistent /.npm
# Switch to non-root user for runtime
USER nobody
# Generate Prisma client as nobody user to ensure correct file ownership
RUN prisma generate
# Prisma runtime knobs for offline containers
ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
PRISMA_HIDE_UPDATE_MESSAGE=1 \

1
docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md
View file

@ -15,6 +15,7 @@ authors:
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Guide to Claude Opus 4.5 and advanced features in LiteLLM: Tool Search, Programmatic Tool Calling, and Effort Parameter."
tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features]
hide_table_of_contents: false
---

1
docs/my-website/blog/gemini_3/index.md
View file

@ -15,6 +15,7 @@ authors:
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK."
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---

1
docs/my-website/blog/gemini_3_flash/index.md
View file

@ -15,6 +15,7 @@ authors:
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support."
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---

92
docs/my-website/blog/sub_millisecond_proxy_overhead/index.md Normal file
View file

@ -0,0 +1,92 @@
---
slug: sub-millisecond-proxy-overhead
title: "Achieving Sub-Millisecond Proxy Overhead"
date: 2026-02-02T10:00:00
authors:
- name: Alexsander Hamir
title: "Performance Engineer, LiteLLM"
url: https://www.linkedin.com/in/alexsander-baptista/
image_url: https://github.com/AlexsanderHamir.png
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Our Q1 performance target and architectural direction for achieving sub-millisecond proxy overhead on modest hardware."
tags: [performance, architecture]
hide_table_of_contents: false
---
![Sidecar architecture: Python control plane vs. sidecar hot path](https://raw.githubusercontent.com/AlexsanderHamir/assets/main/Screenshot%202026-02-02%20172554.png)
# Achieving Sub-Millisecond Proxy Overhead
## Introduction
Our Q1 performance target is to aggressively move toward sub-millisecond proxy overhead on a single instance with 4 CPUs and 8 GB of RAM, and to continue pushing that boundary over time. Our broader goal is to make LiteLLM inexpensive to deploy, lightweight, and fast. This post outlines the architectural direction behind that effort.
Proxy overhead refers to the latency introduced by LiteLLM itself, independent of the upstream provider.
To measure it, we run the same workload directly against the provider and through LiteLLM at identical QPS (for example, 1,000 QPS) and compare the latency delta. To reduce noise, the load generator, LiteLLM, and a mock LLM endpoint all run on the same machine, ensuring the difference reflects proxy overhead rather than network latency.
---
## Where We're Coming From
Under the same benchmark originally conducted by [TensorZero](https://www.tensorzero.com/docs/gateway/benchmarks), LiteLLM previously failed at around 1,000 QPS.
That is no longer the case. Today, LiteLLM can be stress-tested at 1,000 QPS with no failures and can scale up to 5,000 QPS without failures on a 4-CPU, 8-GB RAM single instance setup.
This establishes a more up to date baseline and provides useful context as we continue working on proxy overhead and overall performance.
---
## Design Choice
Achieving sub-millisecond proxy overhead with a Python-based system requires being deliberate about where work happens.
Python is a strong fit for flexibility and extensibility: provider abstraction, configuration-driven routing, and a rich callback ecosystem. These are areas where development velocity and correctness matter more than raw throughput.
At higher request rates, however, certain classes of work become expensive when executed inside the Python process on every request. Rather than rewriting LiteLLM or introducing complex deployment requirements, we adopt an optional **sidecar architecture**.
This architectural change is how we intend to make LiteLLM **permanently fast**. While it supports our near-term performance targets, it is a long-term investment.
Python continues to own:
- Request validation and normalization
- Model and provider selection
- Callbacks and integrations
The sidecar owns **performance-critical execution**, such as:
- Efficient request forwarding
- Connection reuse and pooling
- Enforcing timeouts and limits
- Aggregating high-frequency metrics
This separation allows each component to focus on what it does best: Python acts as the control plane, while the sidecar handles the hot path.
---
### Why the Sidecar Is Optional
The sidecar is intentionally **optional**.
This allows us to ship it incrementally, validate it under real-world workloads, and avoid making it a hard dependency before it is fully battle-tested across all LiteLLM features.
Just as importantly, this ensures that self-hosting LiteLLM remains simple. The sidecar is bundled and started automatically, requires no additional infrastructure, and can be disabled entirely. From a user's perspective, LiteLLM continues to behave like a single service.
As of today, the sidecar is an optimization, not a requirement.
---
## Conclusion
Sub-millisecond proxy overhead is not achieved through a single optimization, but through architectural changes.
By keeping Python focused on orchestration and extensibility, and offloading performance-critical execution to a sidecar, we establish a foundation for making LiteLLM **permanently fast over time**—even on modest hardware such as a 1-CPU, 2-GB RAM instance, while keeping deployment and self-hosting simple.
This work extends beyond Q1, and we will continue sharing benchmarks and updates as the architecture evolves.

227
docs/my-website/docs/a2a.md
View file

@ -68,116 +68,9 @@ Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./pr
## Invoking your Agents
Use the [A2A Python SDK](https://pypi.org/project/a2a/) to invoke agents through LiteLLM.
This example shows how to:
1. **List available agents** - Query `/v1/agents` to see which agents your key can access
2. **Select an agent** - Pick an agent from the list
3. **Invoke via A2A** - Use the A2A protocol to send messages to the agent
```python showLineNumbers title="invoke_a2a_agent.py"
from uuid import uuid4
import httpx
import asyncio
from a2a.client import A2ACardResolver, A2AClient
from a2a.types import MessageSendParams, SendMessageRequest
# === CONFIGURE THESE ===
LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
# =======================
async def main():
headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
async with httpx.AsyncClient(headers=headers) as client:
# Step 1: List available agents
response = await client.get(f"{LITELLM_BASE_URL}/v1/agents")
agents = response.json()
print("Available agents:")
for agent in agents:
print(f" - {agent['agent_name']} (ID: {agent['agent_id']})")
if not agents:
print("No agents available for this key")
return
# Step 2: Select an agent and invoke it
selected_agent = agents[0]
agent_id = selected_agent["agent_id"]
agent_name = selected_agent["agent_name"]
print(f"\nInvoking: {agent_name}")
# Step 3: Use A2A protocol to invoke the agent
base_url = f"{LITELLM_BASE_URL}/a2a/{agent_id}"
resolver = A2ACardResolver(httpx_client=client, base_url=base_url)
agent_card = await resolver.get_agent_card()
a2a_client = A2AClient(httpx_client=client, agent_card=agent_card)
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello, what can you do?"}],
"messageId": uuid4().hex,
}
),
)
response = await a2a_client.send_message(request)
print(f"Response: {response.model_dump(mode='json', exclude_none=True, indent=4)}")
if __name__ == "__main__":
asyncio.run(main())
```
### Streaming Responses
For streaming responses, use `send_message_streaming`:
```python showLineNumbers title="invoke_a2a_agent_streaming.py"
from uuid import uuid4
import httpx
import asyncio
from a2a.client import A2ACardResolver, A2AClient
from a2a.types import MessageSendParams, SendStreamingMessageRequest
# === CONFIGURE THESE ===
LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
LITELLM_AGENT_NAME = "ij-local" # Agent name registered in LiteLLM
# =======================
async def main():
base_url = f"{LITELLM_BASE_URL}/a2a/{LITELLM_AGENT_NAME}"
headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
async with httpx.AsyncClient(headers=headers) as httpx_client:
# Resolve agent card and create client
resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
agent_card = await resolver.get_agent_card()
client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
# Send a streaming message
request = SendStreamingMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello, what can you do?"}],
"messageId": uuid4().hex,
}
),
)
# Stream the response
async for chunk in client.send_message_streaming(request):
print(chunk.model_dump(mode="json", exclude_none=True))
if __name__ == "__main__":
asyncio.run(main())
```
See the [Invoking A2A Agents](./a2a_invoking_agents) guide to learn how to call your agents using:
- **A2A SDK** - Native A2A protocol with full support for tasks and artifacts
- **OpenAI SDK** - Familiar `/chat/completions` interface with `a2a/` model prefix
## Tracking Agent Logs
@ -193,6 +86,120 @@ The logs show:
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
## Forwarding LiteLLM Context Headers
When LiteLLM invokes your A2A agent, it sends special headers that enable:
- **Trace Grouping**: All LLM calls from the same agent execution appear under one trace
- **Agent Spend Tracking**: Costs are attributed to the specific agent
| Header | Purpose |
|--------|---------|
| `X-LiteLLM-Trace-Id` | Links all LLM calls to the same execution flow |
| `X-LiteLLM-Agent-Id` | Attributes spend to the correct agent |
To enable these features, your A2A server must **forward these headers** to any LLM calls it makes back to LiteLLM.
### Implementation Steps
**Step 1: Extract headers from incoming A2A request**
```python def get_litellm_headers(request) -> dict:
"""Extract X-LiteLLM-* headers from incoming A2A request."""
all_headers = request.call_context.state.get('headers', {})
return {
k: v for k, v in all_headers.items()
if k.lower().startswith('x-litellm-')
}
```
**Step 2: Forward headers to your LLM calls**
Pass the extracted headers when making calls back to LiteLLM:
<Tabs>
<TabItem value="openai" label="OpenAI SDK" default>
```python from openai import OpenAI
headers = get_litellm_headers(request)
client = OpenAI(
api_key="sk-your-litellm-key",
base_url="http://localhost:4000",
default_headers=headers, # Forward headers
)
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}]
)
```
</TabItem>
<TabItem value="langchain" label="LangChain">
```python
from langchain_openai import ChatOpenAI
headers = get_litellm_headers(request)
llm = ChatOpenAI(
model="gpt-4o",
openai_api_key="sk-your-litellm-key",
base_url="http://localhost:4000",
default_headers=headers, # Forward headers
)
```
</TabItem>
<TabItem value="litellm" label="LiteLLM SDK">
```python
import litellm
headers = get_litellm_headers(request)
response = litellm.completion(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}],
api_base="http://localhost:4000",
extra_headers=headers, # Forward headers
)
```
</TabItem>
<TabItem value="requests" label="HTTP (requests/httpx)">
```python
import httpx
headers = get_litellm_headers(request)
headers["Authorization"] = "Bearer sk-your-litellm-key"
response = httpx.post(
"http://localhost:4000/v1/chat/completions",
headers=headers,
json={"model": "gpt-4o", "messages": [{"role": "user", "content": "Hello"}]}
)
```
</TabItem>
</Tabs>
### Result
With header forwarding enabled, you'll see:
**Trace Grouping in Langfuse:**
<Image
img={require('../img/a2a_trace_grouping.png')}
style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
/>
**Agent Spend Attribution:**
<Image
img={require('../img/a2a_agent_spend.png')}
style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
/>
## API Reference
### Endpoint

280
docs/my-website/docs/a2a_invoking_agents.md Normal file
View file

@ -0,0 +1,280 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Invoking A2A Agents
Learn how to invoke A2A agents through LiteLLM using different methods.
:::tip Deploy Your Own A2A Agent
Want to test with your own agent? Deploy this template A2A agent powered by Google Gemini:
[**shin-bot-litellm/a2a-gemini-agent**](https://github.com/shin-bot-litellm/a2a-gemini-agent) - Simple deployable A2A agent with streaming support
:::
## A2A SDK
Use the [A2A Python SDK](https://pypi.org/project/a2a-sdk) to invoke agents through LiteLLM using the A2A protocol.
### Non-Streaming
This example shows how to:
1. **List available agents** - Query `/v1/agents` to see which agents your key can access
2. **Select an agent** - Pick an agent from the list
3. **Invoke via A2A** - Use the A2A protocol to send messages to the agent
```python showLineNumbers title="invoke_a2a_agent.py"
from uuid import uuid4
import httpx
import asyncio
from a2a.client import A2ACardResolver, A2AClient
from a2a.types import MessageSendParams, SendMessageRequest
# === CONFIGURE THESE ===
LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
# =======================
async def main():
headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
async with httpx.AsyncClient(headers=headers) as client:
# Step 1: List available agents
response = await client.get(f"{LITELLM_BASE_URL}/v1/agents")
agents = response.json()
print("Available agents:")
for agent in agents:
print(f" - {agent['agent_name']} (ID: {agent['agent_id']})")
if not agents:
print("No agents available for this key")
return
# Step 2: Select an agent and invoke it
selected_agent = agents[0]
agent_id = selected_agent["agent_id"]
agent_name = selected_agent["agent_name"]
print(f"\nInvoking: {agent_name}")
# Step 3: Use A2A protocol to invoke the agent
base_url = f"{LITELLM_BASE_URL}/a2a/{agent_id}"
resolver = A2ACardResolver(httpx_client=client, base_url=base_url)
agent_card = await resolver.get_agent_card()
a2a_client = A2AClient(httpx_client=client, agent_card=agent_card)
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello, what can you do?"}],
"messageId": uuid4().hex,
}
),
)
response = await a2a_client.send_message(request)
print(f"Response: {response.model_dump(mode='json', exclude_none=True, indent=4)}")
if __name__ == "__main__":
asyncio.run(main())
```
### Streaming
For streaming responses, use `send_message_streaming`:
```python showLineNumbers title="invoke_a2a_agent_streaming.py"
from uuid import uuid4
import httpx
import asyncio
from a2a.client import A2ACardResolver, A2AClient
from a2a.types import MessageSendParams, SendStreamingMessageRequest
# === CONFIGURE THESE ===
LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
LITELLM_AGENT_NAME = "ij-local" # Agent name registered in LiteLLM
# =======================
async def main():
base_url = f"{LITELLM_BASE_URL}/a2a/{LITELLM_AGENT_NAME}"
headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
async with httpx.AsyncClient(headers=headers) as httpx_client:
# Resolve agent card and create client
resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
agent_card = await resolver.get_agent_card()
client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
# Send a streaming message
request = SendStreamingMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Tell me a long story"}],
"messageId": uuid4().hex,
}
),
)
# Stream the response
async for chunk in client.send_message_streaming(request):
print(chunk.model_dump(mode="json", exclude_none=True))
if __name__ == "__main__":
asyncio.run(main())
```
## /chat/completions API (OpenAI SDK)
You can also invoke A2A agents using the familiar OpenAI SDK by using the `a2a/` model prefix.
### Non-Streaming
<Tabs>
<TabItem value="python" label="Python" default>
```python showLineNumbers title="openai_non_streaming.py"
import openai
client = openai.OpenAI(
api_key="sk-1234", # Your LiteLLM Virtual Key
base_url="http://localhost:4000" # Your LiteLLM proxy URL
)
response = client.chat.completions.create(
model="a2a/my-agent", # Use a2a/ prefix with your agent name
messages=[
{"role": "user", "content": "Hello, what can you do?"}
]
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="typescript" label="TypeScript">
```typescript showLineNumbers title="openai_non_streaming.ts"
import OpenAI from 'openai';
const client = new OpenAI({
apiKey: 'sk-1234', // Your LiteLLM Virtual Key
baseURL: 'http://localhost:4000' // Your LiteLLM proxy URL
});
const response = await client.chat.completions.create({
model: 'a2a/my-agent', // Use a2a/ prefix with your agent name
messages: [
{ role: 'user', content: 'Hello, what can you do?' }
]
});
console.log(response.choices[0].message.content);
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="curl_non_streaming.sh"
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "a2a/my-agent",
"messages": [
{"role": "user", "content": "Hello, what can you do?"}
]
}'
```
</TabItem>
</Tabs>
### Streaming
<Tabs>
<TabItem value="python" label="Python" default>
```python showLineNumbers title="openai_streaming.py"
import openai
client = openai.OpenAI(
api_key="sk-1234", # Your LiteLLM Virtual Key
base_url="http://localhost:4000" # Your LiteLLM proxy URL
)
stream = client.chat.completions.create(
model="a2a/my-agent", # Use a2a/ prefix with your agent name
messages=[
{"role": "user", "content": "Tell me a long story"}
],
stream=True
)
for chunk in stream:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="", flush=True)
```
</TabItem>
<TabItem value="typescript" label="TypeScript">
```typescript showLineNumbers title="openai_streaming.ts"
import OpenAI from 'openai';
const client = new OpenAI({
apiKey: 'sk-1234', // Your LiteLLM Virtual Key
baseURL: 'http://localhost:4000' // Your LiteLLM proxy URL
});
const stream = await client.chat.completions.create({
model: 'a2a/my-agent', // Use a2a/ prefix with your agent name
messages: [
{ role: 'user', content: 'Tell me a long story' }
],
stream: true
});
for await (const chunk of stream) {
const content = chunk.choices[0]?.delta?.content;
if (content) {
process.stdout.write(content);
}
}
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="curl_streaming.sh"
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "a2a/my-agent",
"messages": [
{"role": "user", "content": "Tell me a long story"}
],
"stream": true
}'
```
</TabItem>
</Tabs>
## Key Differences
| Method | Use Case | Advantages |
|--------|----------|------------|
| **A2A SDK** | Native A2A protocol integration | • Full A2A protocol support<br/>• Access to task states and artifacts<br/>• Context management |
| **OpenAI SDK** | Familiar OpenAI-style interface | • Drop-in replacement for OpenAI calls<br/>• Easier migration from LLM to agent workflows<br/>• Works with existing OpenAI tooling |
:::tip Model Prefix
When using the OpenAI SDK, always prefix your agent name with `a2a/` (e.g., `a2a/my-agent`) to route requests to the A2A agent instead of an LLM provider.
:::

7
docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md
View file

@ -101,12 +101,11 @@ model_list:
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
guardrails:
guardrails:
- guardrail_name: my_guardrail
litellm_params:
litellm_params:
guardrail: my_guardrail
mode: during_call
api_key: os.environ/MY_GUARDRAIL_API_KEY

22
docs/my-website/docs/benchmarks.md
View file

@ -48,6 +48,28 @@ In these tests the baseline latency characteristics are measured against a fake-
- High-percentile latencies drop significantly: P95 630 ms → 150 ms, P99 1,200 ms → 240 ms.
- Setting workers equal to CPU count gives optimal performance.
## `/realtime` API Benchmarks
End-to-end latency benchmarks for the `/realtime` endpoint tested against a fake realtime endpoint.
### Performance Metrics
| Metric | Value |
| --------------- | ---------- |
| Median latency | 59 ms |
| p95 latency | 67 ms |
| p99 latency | 99 ms |
| Average latency | 63 ms |
| RPS | 1,207 |
### Test Setup
| Category | Specification |
|----------|---------------|
| **Load Testing** | Locust: 1,000 concurrent users, 500 ramp-up |
| **System** | 4 vCPUs, 8 GB RAM, 4 workers, 4 instances |
| **Database** | PostgreSQL (Redis unused) |
## Machine Spec used for testing
Each machine deploying LiteLLM had the following specs:

12
docs/my-website/docs/enterprise.md
View file

@ -74,6 +74,18 @@ You can find [supported data regions litellm here](../docs/data_security#support
## Frequently Asked Questions
### How to set up and verify your Enterprise License
1. Add your license key to the environment:
```env
LITELLM_LICENSE="eyJ..."
```
2. Restart LiteLLM Proxy.
3. Open `http://<your-proxy-host>:<port>/` — the Swagger page should show **"Enterprise Edition"** in the description. If it doesn't, check that the key is correct, unexpired, and that the proxy was fully restarted.
### SLA's + Professional Support
Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting. We can’t solve your own infrastructure-related issues but we will guide you to fix them.

158
docs/my-website/docs/mcp_semantic_filter.md Normal file
View file

@ -0,0 +1,158 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# MCP Semantic Tool Filter
Automatically filter MCP tools by semantic relevance. When you have many MCP tools registered, LiteLLM semantically matches the user's query against tool descriptions and sends only the most relevant tools to the LLM.
## How It Works
Tool search shifts tool selection from a prompt-engineering problem to a retrieval problem. Instead of injecting a large static list of tools into every prompt, the semantic filter:
1. Builds a semantic index of all available MCP tools on startup
2. On each request, semantically matches the user's query against tool descriptions
3. Returns only the top-K most relevant tools to the LLM
This approach improves context efficiency, increases reliability by reducing tool confusion, and enables scalability to ecosystems with hundreds or thousands of MCP tools.
```mermaid
sequenceDiagram
participant Client
participant LiteLLM as LiteLLM Proxy
participant SemanticFilter as Semantic Filter
participant MCP as MCP Registry
participant LLM as LLM Provider
Note over LiteLLM,MCP: Startup: Build Semantic Index
LiteLLM->>MCP: Fetch all registered MCP tools
MCP->>LiteLLM: Return all tools (e.g., 50 tools)
LiteLLM->>SemanticFilter: Build semantic router with embeddings
SemanticFilter->>LLM: Generate embeddings for tool descriptions
LLM->>SemanticFilter: Return embeddings
Note over SemanticFilter: Index ready for fast lookup
Note over Client,LLM: Request: Semantic Tool Filtering
Client->>LiteLLM: POST /v1/responses with MCP tools
LiteLLM->>SemanticFilter: Expand MCP references (50 tools available)
SemanticFilter->>SemanticFilter: Extract user query from request
SemanticFilter->>LLM: Generate query embedding
LLM->>SemanticFilter: Return query embedding
SemanticFilter->>SemanticFilter: Match query against tool embeddings
SemanticFilter->>LiteLLM: Return top-K tools (e.g., 3 most relevant)
LiteLLM->>LLM: Forward request with filtered tools (3 tools)
LLM->>LiteLLM: Return response
LiteLLM->>Client: Response with headers<br/>x-litellm-semantic-filter: 50->3<br/>x-litellm-semantic-filter-tools: tool1,tool2,tool3
```
## Configuration
Enable semantic filtering in your LiteLLM config:
```yaml title="config.yaml" showLineNumbers
litellm_settings:
mcp_semantic_tool_filter:
enabled: true
embedding_model: "text-embedding-3-small" # Model for semantic matching
top_k: 5 # Max tools to return
similarity_threshold: 0.3 # Min similarity score
```
**Configuration Options:**
- `enabled` - Enable/disable semantic filtering (default: `false`)
- `embedding_model` - Model for generating embeddings (default: `"text-embedding-3-small"`)
- `top_k` - Maximum number of tools to return (default: `10`)
- `similarity_threshold` - Minimum similarity score for matches (default: `0.3`)
## Usage
Use MCP tools normally with the Responses API or Chat Completions. The semantic filter runs automatically:
<Tabs>
<TabItem value="responses" label="Responses API">
```bash title="Responses API with Semantic Filtering" showLineNumbers
curl --location 'http://localhost:4000/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer sk-1234" \
--data '{
"model": "gpt-4o",
"input": [
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
}
],
"tools": [
{
"type": "mcp",
"server_url": "litellm_proxy",
"require_approval": "never"
}
],
"tool_choice": "required"
}'
```
</TabItem>
<TabItem value="chat" label="Chat Completions">
```bash title="Chat Completions with Semantic Filtering" showLineNumbers
curl --location 'http://localhost:4000/v1/chat/completions' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer sk-1234" \
--data '{
"model": "gpt-4o",
"messages": [
{"role": "user", "content": "Search Wikipedia for LiteLLM"}
],
"tools": [
{
"type": "mcp",
"server_url": "litellm_proxy"
}
]
}'
```
</TabItem>
</Tabs>
## Response Headers
The semantic filter adds diagnostic headers to every response:
```
x-litellm-semantic-filter: 10->3
x-litellm-semantic-filter-tools: wikipedia-fetch,github-search,slack-post
```
- **`x-litellm-semantic-filter`** - Shows before→after tool count (e.g., `10->3` means 10 tools were filtered down to 3)
- **`x-litellm-semantic-filter-tools`** - CSV list of the filtered tool names (max 150 chars, clipped with `...` if longer)
These headers help you understand which tools were selected for each request and verify the filter is working correctly.
## Example
If you have 50 MCP tools registered and make a request asking about Wikipedia, the semantic filter will:
1. Semantically match your query `"Search Wikipedia for LiteLLM"` against all 50 tool descriptions
2. Select the top 5 most relevant tools (e.g., `wikipedia-fetch`, `wikipedia-search`, etc.)
3. Pass only those 5 tools to the LLM
4. Add headers showing `x-litellm-semantic-filter: 50->5`
This dramatically reduces prompt size while ensuring the LLM has access to the right tools for the task.
## Performance
The semantic filter is optimized for production:
- Router builds once on startup (no per-request overhead)
- Semantic matching typically takes under 50ms
- Fails gracefully - returns all tools if filtering fails
- No impact on latency for requests without MCP tools
## Related
- [MCP Overview](./mcp.md) - Learn about MCP in LiteLLM
- [MCP Permission Management](./mcp_control.md) - Control tool access by key/team
- [Using MCP](./mcp_usage.md) - Complete MCP usage guide

52
docs/my-website/docs/observability/datadog.md
View file

@ -7,6 +7,7 @@ import TabItem from '@theme/TabItem';
LiteLLM Supports logging to the following Datdog Integrations:
- `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
- `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
- `datadog_cost_management` [Datadog Cloud Cost Management](#datadog-cloud-cost-management)
- `ddtrace-run` [Datadog Tracing](#datadog-tracing)
## Datadog Logs
@ -73,7 +74,7 @@ Send logs through a local DataDog agent (useful for containerized environments):
```shell
LITELLM_DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent
LITELLM_DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518)
DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth)
DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (Agent handles auth for Logs. REQUIRED for LLM Observability)
DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source
```
@ -84,6 +85,9 @@ When `LITELLM_DD_AGENT_HOST` is set, logs are sent to the agent instead of direc
**Note:** We use `LITELLM_DD_AGENT_HOST` instead of `DD_AGENT_HOST` to avoid conflicts with `ddtrace` which automatically sets `DD_AGENT_HOST` for APM tracing.
> [!IMPORTANT]
> **Datadog LLM Observability**: `DD_API_KEY` is **REQUIRED** even when using the Datadog Agent (`LITELLM_DD_AGENT_HOST`). The agent acts as a proxy but the API key header is mandatory for the LLM Observability endpoint.
**Step 3**: Start the proxy, make a test request
Start proxy
@ -161,6 +165,50 @@ On the Datadog LLM Observability page, you should see that both input messages a
<Image img={require('../../img/dd_llm_obs.png')} />
## Datadog Cloud Cost Management
| Feature | Details |
|---------|---------|
| **What is logged** | Aggregated LLM Costs (FOCUS format) |
| **Events** | Periodic Uploads of Aggregated Cost Data |
| **Product Link** | [Datadog Cloud Cost Management](https://docs.datadoghq.com/cost_management/) |
We will use the `--config` to set `litellm.callbacks = ["datadog_cost_management"]`. This will periodically upload aggregated LLM cost data to Datadog.
**Step 1**: Create a `config.yaml` file and set `litellm_settings`: `success_callback`
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
litellm_settings:
callbacks: ["datadog_cost_management"]
```
**Step 2**: Set Required env variables
```shell
DD_API_KEY="your-api-key"
DD_APP_KEY="your-app-key" # REQUIRED for Cost Management
DD_SITE="us5.datadoghq.com"
```
**Step 3**: Start the proxy
```shell
litellm --config config.yaml
```
**How it works**
* LiteLLM aggregates costs in-memory by Provider, Model, Date, and Tags.
* Requires `DD_APP_KEY` for the Custom Costs API.
* Costs are uploaded periodically (flushed).
### Datadog Tracing
Use `ddtrace-run` to enable [Datadog Tracing](https://ddtrace.readthedocs.io/en/stable/installation_quickstart.html) on litellm proxy
@ -203,5 +251,5 @@ LiteLLM supports customizing the following Datadog environment variables
| `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No |
\* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required
\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required
\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required for **Datadog Logs**. (**Note: `DD_API_KEY` IS REQUIRED for Datadog LLM Observability**)

60
docs/my-website/docs/observability/langfuse_integration.md
View file

@ -215,6 +215,66 @@ The following parameters can be updated on a continuation of a trace by passing
Any other key value pairs passed into the metadata not listed in the above spec for a `litellm` completion will be added as a metadata key value pair for the generation.
#### Multiple Langfuse Projects (Per-Request Credentials)
You can send traces to different Langfuse projects per request by passing credentials directly to `completion()` or `acompletion()`. This works alongside (or instead of) the global env vars and is useful when different teams or business processes use different Langfuse projects.
Pass **`langfuse_public_key`**, **`langfuse_secret_key`** (or **`langfuse_secret`**), and optionally **`langfuse_host`** as keyword arguments:
```python
import litellm
from litellm import completion
# Optional: set a default via env for requests that don't pass credentials
# os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-default..."
# os.environ["LANGFUSE_SECRET_KEY"] = "sk-default..."
litellm.success_callback = ["langfuse"]
litellm.failure_callback = ["langfuse"]
# Request 1 → Langfuse Project A
response_a = completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello from team A"}],
langfuse_public_key="pk-lf-project-a...",
langfuse_secret_key="sk-lf-project-a...",
langfuse_host="https://us.cloud.langfuse.com", # optional
)
# Request 2 → Langfuse Project B (different project)
response_b = completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello from team B"}],
langfuse_public_key="pk-lf-project-b...",
langfuse_secret_key="sk-lf-project-b...",
langfuse_host="https://eu.cloud.langfuse.com", # optional, can differ per project
)
```
Async usage with per-request credentials:
```python
import litellm
from litellm import acompletion
litellm.success_callback = ["langfuse"]
litellm.failure_callback = ["langfuse"]
response = await acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hi"}],
langfuse_public_key="pk-lf-...",
langfuse_secret_key="sk-lf-...",
langfuse_host="https://us.cloud.langfuse.com", # optional
)
```
- **`langfuse_public_key`** – Langfuse project public key (required for per-request override).
- **`langfuse_secret_key`** or **`langfuse_secret`** – Langfuse secret key (either name is accepted).
- **`langfuse_host`** – Langfuse host URL (e.g. `https://us.cloud.langfuse.com`); optional, defaults to env or Langfuse cloud.
When these are passed, that request uses this project (and host) for the Langfuse callback; when omitted, the callback uses the global Langfuse client (from env vars if set). LiteLLM caches a Langfuse client per credential set to avoid creating a new client on every request.
#### Disable Logging - Specific Calls
To disable logging for specific calls use the `no-log` flag.

25
docs/my-website/docs/pass_through/openai_passthrough.md
View file

@ -1,6 +1,6 @@
# OpenAI Passthrough
Pass-through endpoints for `/openai`
Pass-through endpoints for direct OpenAI API access
## Overview
@ -10,12 +10,27 @@ Pass-through endpoints for `/openai`
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | Fully supported |
### When to use this?
## Available Endpoints
### `/openai_passthrough` - Recommended
Dedicated passthrough endpoint that guarantees direct routing to OpenAI without conflicts.
**Use this for:**
- OpenAI Responses API (`/v1/responses`)
- Any endpoint where you need guaranteed passthrough
- When `/openai` routes are conflicting with LiteLLM's native implementations
### `/openai` - Legacy
Standard passthrough endpoint that may conflict with LiteLLM's native implementations.
**Note:** Some endpoints like `/openai/v1/responses` will be routed to LiteLLM's native implementation instead of OpenAI.
## When to use this?
- For 90% of your use cases, you should use the [native LiteLLM OpenAI Integration](https://docs.litellm.ai/docs/providers/openai) (`/chat/completions`, `/embeddings`, `/completions`, `/images`, `/batches`, etc.)
- Use this passthrough to call less popular or newer OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`
- Use `/openai_passthrough` to call less popular or newer OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`, `/responses`
Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai`
Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai_passthrough`
## Usage Examples
@ -34,7 +49,7 @@ Make sure you do the following:
import openai
client = openai.OpenAI(
base_url="http://0.0.0.0:4000/openai", # <your-proxy-url>/openai
base_url="http://0.0.0.0:4000/openai_passthrough", # <your-proxy-url>/openai_passthrough
api_key="sk-anything" # <your-proxy-api-key>
)
```

163
docs/my-website/docs/providers/azure_ai/azure_model_router.md
View file

@ -5,19 +5,38 @@ Azure Model Router is a feature in Azure AI Foundry that automatically routes yo
## Key Features
- **Automatic Model Selection**: Azure Model Router dynamically selects the best model for your request
- **Cost Tracking**: LiteLLM automatically tracks costs based on the actual model used (e.g., `gpt-4.1-nano`), not the router endpoint
- **Cost Tracking**: LiteLLM automatically tracks costs based on the actual model used (e.g., `gpt-4.1-nano`), plus the Model Router infrastructure fee
- **Streaming Support**: Full support for streaming responses with accurate cost calculation
- **Simple Configuration**: Easy to set up via UI or config file
## Model Naming Pattern
Use the pattern: `azure_ai/model_router/<deployment-name>`
**Components:**
- `azure_ai` - The provider identifier
- `model_router` - Indicates this is a Model Router deployment
- `<deployment-name>` - Your actual deployment name from Azure AI Foundry (e.g., `azure-model-router`)
**Example:** `azure_ai/model_router/azure-model-router`
**How it works:**
- LiteLLM automatically strips the `model_router/` prefix when sending requests to Azure
- Only your deployment name (e.g., `azure-model-router`) is sent to the Azure API
- The full path is preserved in responses and logs for proper cost tracking
## LiteLLM Python SDK
### Basic Usage
Use the pattern `azure_ai/model_router/<deployment-name>` where `<deployment-name>` is your Azure deployment name:
```python
import litellm
import os
response = litellm.completion(
model="azure_ai/azure-model-router",
model="azure_ai/model_router/azure-model-router", # Use your deployment name
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
@ -26,6 +45,13 @@ response = litellm.completion(
print(response)
```
**Pattern Explanation:**
- `azure_ai` - The provider
- `model_router` - Indicates this is a model router deployment
- `azure-model-router` - Your actual deployment name from Azure AI Foundry
LiteLLM will automatically strip the `model_router/` prefix when sending the request to Azure, so only `azure-model-router` is sent to the API.
### Streaming with Usage Tracking
```python
@ -33,7 +59,7 @@ import litellm
import os
response = await litellm.acompletion(
model="azure_ai/azure-model-router",
model="azure_ai/model_router/azure-model-router", # Use your deployment name
messages=[{"role": "user", "content": "hi"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
@ -51,13 +77,15 @@ async for chunk in response:
```yaml
model_list:
- model_name: azure-model-router
- model_name: azure-model-router # Public name for your users
litellm_params:
model: azure_ai/azure-model-router
model: azure_ai/model_router/azure-model-router # Use your deployment name
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/v1/
api_key: os.environ/AZURE_MODEL_ROUTER_API_KEY
```
**Note:** Replace `azure-model-router` in the model path with your actual deployment name from Azure AI Foundry.
### Start Proxy
```bash
@ -80,49 +108,42 @@ curl -X POST http://localhost:4000/chat/completions \
This walkthrough shows how to add an Azure Model Router endpoint to LiteLLM using the Admin Dashboard.
### Select Provider
### Quick Start
1. Navigate to the **Models** page in the LiteLLM UI
2. Select **"Azure AI Foundry (Studio)"** as the provider
3. Enter your deployment name (e.g., `azure-model-router`)
4. LiteLLM will automatically format it as `azure_ai/model_router/azure-model-router`
5. Add your API base URL and API key
6. Test and save
### Detailed Walkthrough
#### Step 1: Select Provider
Navigate to the Models page and select "Azure AI Foundry (Studio)" as the provider.
#### Navigate to Models Page
##### Navigate to Models Page
![Navigate to Models](./img/azure_model_router_01.jpeg)
#### Click Provider Dropdown
##### Click Provider Dropdown
![Click Provider](./img/azure_model_router_02.jpeg)
#### Choose Azure AI Foundry
##### Choose Azure AI Foundry
![Select Azure AI Foundry](./img/azure_model_router_03.jpeg)
### Configure Model Name
#### Step 2: Enter Deployment Name
Set up the model name by entering `azure_ai/` followed by your model router deployment name from Azure.
**New Simplified Method:** Just enter your deployment name directly in the text field. If your deployment name contains "model-router" or "model_router", LiteLLM will automatically format it as `azure_ai/model_router/<deployment-name>`.
#### Click Model Name Field
**Example:**
- Enter: `azure-model-router`
- LiteLLM creates: `azure_ai/model_router/azure-model-router`
![Click Model Field](./img/azure_model_router_04.jpeg)
#### Select Custom Model Name
![Select Custom Model](./img/azure_model_router_05.jpeg)
#### Enter LiteLLM Model Name
![LiteLLM Model Name](./img/azure_model_router_06.jpeg)
#### Click Custom Model Name Field
![Enter Custom Name Field](./img/azure_model_router_07.jpeg)
#### Type Model Prefix
Type `azure_ai/` as the prefix.
![Type azure_ai prefix](./img/azure_model_router_08.jpeg)
#### Copy Model Name from Azure Portal
##### Copy Deployment Name from Azure Portal
Switch to Azure AI Foundry and copy your model router deployment name.
@ -130,73 +151,79 @@ Switch to Azure AI Foundry and copy your model router deployment name.
![Copy Model Name](./img/azure_model_router_10.jpeg)
#### Paste Model Name
##### Enter Deployment Name in LiteLLM
Paste to get `azure_ai/azure-model-router`.
Paste your deployment name (e.g., `azure-model-router`) directly into the text field.
![Paste Model Name](./img/azure_model_router_11.jpeg)
![Enter Deployment Name](./img/azure_model_router_04.jpeg)
### Configure API Base and Key
**What happens behind the scenes:**
- You enter: `azure-model-router`
- LiteLLM automatically detects this is a model router deployment
- The full model path becomes: `azure_ai/model_router/azure-model-router`
- When making API calls, only `azure-model-router` is sent to Azure
#### Step 3: Configure API Base and Key
Copy the endpoint URL and API key from Azure portal.
#### Copy API Base URL from Azure
##### Copy API Base URL from Azure
![Copy API Base](./img/azure_model_router_12.jpeg)
#### Enter API Base in LiteLLM
##### Enter API Base in LiteLLM
![Click API Base Field](./img/azure_model_router_13.jpeg)
![Paste API Base](./img/azure_model_router_14.jpeg)
#### Copy API Key from Azure
##### Copy API Key from Azure
![Copy API Key](./img/azure_model_router_15.jpeg)
#### Enter API Key in LiteLLM
##### Enter API Key in LiteLLM
![Enter API Key](./img/azure_model_router_16.jpeg)
### Test and Add Model
#### Step 4: Test and Add Model
Verify your configuration works and save the model.
#### Test Connection
##### Test Connection
![Test Connection](./img/azure_model_router_17.jpeg)
#### Close Test Dialog
##### Close Test Dialog
![Close Dialog](./img/azure_model_router_18.jpeg)
#### Add Model
##### Add Model
![Add Model](./img/azure_model_router_19.jpeg)
### Verify in Playground
#### Step 5: Verify in Playground
Test your model and verify cost tracking is working.
#### Open Playground
##### Open Playground
![Go to Playground](./img/azure_model_router_20.jpeg)
#### Select Model
##### Select Model
![Select Model](./img/azure_model_router_21.jpeg)
#### Send Test Message
##### Send Test Message
![Send Message](./img/azure_model_router_22.jpeg)
#### View Logs
##### View Logs
![View Logs](./img/azure_model_router_23.jpeg)
#### Verify Cost Tracking
##### Verify Cost Tracking
Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`).
Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`), plus a flat infrastructure cost of $0.14 per million input tokens for using the Model Router.
![Verify Cost](./img/azure_model_router_24.jpeg)
@ -205,28 +232,50 @@ Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`).
LiteLLM automatically handles cost tracking for Azure Model Router by:
1. **Detecting the actual model**: When Azure Model Router routes your request to a specific model (e.g., `gpt-4.1-nano-2025-04-14`), LiteLLM extracts this from the response
2. **Calculating accurate costs**: Costs are calculated based on the actual model used, not the router endpoint name
2. **Calculating accurate costs**: Costs are calculated based on:
- The actual model used (e.g., `gpt-4.1-nano` token costs)
- Plus a flat infrastructure cost of **$0.14 per million input tokens** for using the Model Router
3. **Streaming support**: Cost tracking works correctly for both streaming and non-streaming requests
### Cost Breakdown
When you use Azure Model Router, the total cost includes:
- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-4.1-nano`)
- **Router Flat Cost**: $0.14 per million input tokens (Azure AI Foundry infrastructure fee)
### Example Response with Cost
```python
import litellm
response = litellm.completion(
model="azure_ai/azure-model-router",
model="azure_ai/model_router/azure-model-router",
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key="your-api-key",
)
# The response will show the actual model used
print(f"Model used: {response.model}") # e.g., "gpt-4.1-nano-2025-04-14"
print(f"Model used: {response.model}") # e.g., "azure_ai/gpt-4.1-nano-2025-04-14"
# Get cost
# Get cost (includes both model cost and router flat cost)
from litellm import completion_cost
cost = completion_cost(completion_response=response)
print(f"Cost: ${cost}")
print(f"Total cost: ${cost}")
# Access detailed cost breakdown
if hasattr(response, '_hidden_params') and 'response_cost' in response._hidden_params:
print(f"Response cost: ${response._hidden_params['response_cost']}")
```
### Viewing Cost Breakdown in UI
When viewing logs in the LiteLLM UI, you'll see:
- **Model Cost**: The cost for the actual model used
- **Azure Model Router Flat Cost**: The $0.14/M input tokens infrastructure fee
- **Total Cost**: Sum of both costs
This breakdown helps you understand exactly what you're paying for when using the Model Router.

2
docs/my-website/docs/providers/bedrock.md
View file

@ -9,7 +9,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor
| Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc), [`bedrock/moonshot`](./bedrock_imported.md#moonshot-kimi-k2-thinking) |
| Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations`, `/v1/realtime`|
| Rerank Endpoint | `/rerank` |
| Pass-through Endpoint | [Supported](../pass_through/bedrock.md) |

362
docs/my-website/docs/providers/bedrock_realtime_with_audio.md Normal file
View file

@ -0,0 +1,362 @@
# Bedrock Realtime API
## Overview
Amazon Bedrock's Nova Sonic model supports real-time bidirectional audio streaming for voice conversations. This tutorial shows how to use it through LiteLLM Proxy.
## Setup
### 1. Configure LiteLLM Proxy
Create a `config.yaml` file:
```yaml
model_list:
- model_name: "bedrock-sonic"
litellm_params:
model: bedrock/amazon.nova-sonic-v1:0
aws_region_name: us-east-1 # or your preferred region
model_info:
mode: realtime
```
### 2. Start LiteLLM Proxy
```bash
litellm --config config.yaml
```
## Basic Text Interaction
```python
import asyncio
import websockets
import json
LITELLM_API_KEY = "sk-1234" # Your LiteLLM API key
LITELLM_URL = 'ws://localhost:4000/v1/realtime?model=bedrock-sonic'
async def test_text_conversation():
async with websockets.connect(
LITELLM_URL,
additional_headers={
"Authorization": f"Bearer {LITELLM_API_KEY}"
}
) as ws:
# Wait for session.created
response = await ws.recv()
print(f"Connected: {json.loads(response)['type']}")
# Configure session
session_update = {
"type": "session.update",
"session": {
"instructions": "You are a helpful assistant.",
"modalities": ["text"],
"temperature": 0.8
}
}
await ws.send(json.dumps(session_update))
# Send a message
message = {
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "Hello!"}]
}
}
await ws.send(json.dumps(message))
# Trigger response
await ws.send(json.dumps({"type": "response.create"}))
# Listen for response
while True:
response = await ws.recv()
event = json.loads(response)
if event['type'] == 'response.text.delta':
print(event['delta'], end='', flush=True)
elif event['type'] == 'response.done':
print("\n✓ Complete")
break
if __name__ == "__main__":
asyncio.run(test_text_conversation())
```
## Audio Streaming with Voice Conversation
```python
import asyncio
import websockets
import json
import base64
import pyaudio
LITELLM_API_KEY = "sk-1234"
LITELLM_URL = 'ws://localhost:4000/v1/realtime?model=bedrock-sonic'
# Audio configuration
INPUT_RATE = 16000 # Nova Sonic expects 16kHz input
OUTPUT_RATE = 24000 # Nova Sonic outputs 24kHz
CHUNK = 1024
async def audio_conversation():
# Initialize PyAudio
p = pyaudio.PyAudio()
# Input stream (microphone)
input_stream = p.open(
format=pyaudio.paInt16,
channels=1,
rate=INPUT_RATE,
input=True,
frames_per_buffer=CHUNK
)
# Output stream (speakers)
output_stream = p.open(
format=pyaudio.paInt16,
channels=1,
rate=OUTPUT_RATE,
output=True,
frames_per_buffer=CHUNK
)
async with websockets.connect(
LITELLM_URL,
additional_headers={"Authorization": f"Bearer {LITELLM_API_KEY}"}
) as ws:
# Wait for session.created
await ws.recv()
print("✓ Connected")
# Configure session with audio
session_update = {
"type": "session.update",
"session": {
"instructions": "You are a friendly voice assistant.",
"modalities": ["text", "audio"],
"voice": "matthew",
"input_audio_format": "pcm16",
"output_audio_format": "pcm16"
}
}
await ws.send(json.dumps(session_update))
print("🎤 Speak into your microphone...")
async def send_audio():
"""Capture and send audio from microphone"""
while True:
audio_data = input_stream.read(CHUNK, exception_on_overflow=False)
audio_b64 = base64.b64encode(audio_data).decode('utf-8')
await ws.send(json.dumps({
"type": "input_audio_buffer.append",
"audio": audio_b64
}))
await asyncio.sleep(0.01)
async def receive_audio():
"""Receive and play audio responses"""
while True:
response = await ws.recv()
event = json.loads(response)
if event['type'] == 'response.audio.delta':
audio_b64 = event.get('delta', '')
if audio_b64:
audio_bytes = base64.b64decode(audio_b64)
output_stream.write(audio_bytes)
elif event['type'] == 'response.text.delta':
print(event['delta'], end='', flush=True)
elif event['type'] == 'response.done':
print("\n✓ Response complete")
# Run both tasks concurrently
await asyncio.gather(send_audio(), receive_audio())
if __name__ == "__main__":
try:
asyncio.run(audio_conversation())
except KeyboardInterrupt:
print("\n\nGoodbye!")
```
## Using Tools/Function Calling
```python
import asyncio
import websockets
import json
from datetime import datetime
LITELLM_API_KEY = "sk-1234"
LITELLM_URL = 'ws://localhost:4000/v1/realtime?model=bedrock-sonic'
# Define tools
TOOLS = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather for a location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "City name"
}
},
"required": ["location"]
}
}
}
]
def get_weather(location: str) -> dict:
"""Simulated weather function"""
return {
"location": location,
"temperature": 72,
"conditions": "sunny"
}
async def conversation_with_tools():
async with websockets.connect(
LITELLM_URL,
additional_headers={"Authorization": f"Bearer {LITELLM_API_KEY}"}
) as ws:
# Wait for session.created
await ws.recv()
# Configure session with tools
session_update = {
"type": "session.update",
"session": {
"instructions": "You are a helpful assistant with access to tools.",
"modalities": ["text"],
"tools": TOOLS
}
}
await ws.send(json.dumps(session_update))
# Send a message that requires a tool
message = {
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "What's the weather in San Francisco?"}]
}
}
await ws.send(json.dumps(message))
await ws.send(json.dumps({"type": "response.create"}))
# Handle responses and tool calls
while True:
response = await ws.recv()
event = json.loads(response)
if event['type'] == 'response.text.delta':
print(event['delta'], end='', flush=True)
elif event['type'] == 'response.function_call_arguments.done':
# Execute the tool
function_name = event['name']
arguments = json.loads(event['arguments'])
print(f"\n🔧 Calling {function_name}({arguments})")
result = get_weather(**arguments)
# Send tool result back
tool_result = {
"type": "conversation.item.create",
"item": {
"type": "function_call_output",
"call_id": event['call_id'],
"output": json.dumps(result)
}
}
await ws.send(json.dumps(tool_result))
await ws.send(json.dumps({"type": "response.create"}))
elif event['type'] == 'response.done':
print("\n✓ Complete")
break
if __name__ == "__main__":
asyncio.run(conversation_with_tools())
```
## Configuration Options
### Voice Options
Available voices: `matthew`, `joanna`, `ruth`, `stephen`, `gregory`, `amy`
### Audio Formats
- **Input**: 16kHz PCM16 (mono)
- **Output**: 24kHz PCM16 (mono)
### Modalities
- `["text"]` - Text only
- `["audio"]` - Audio only
- `["text", "audio"]` - Both text and audio
## Example Test Scripts
Complete working examples are available in the LiteLLM repository:
- **Basic audio streaming**: `test_bedrock_realtime_client.py`
- **Simple text test**: `test_bedrock_realtime_simple.py`
- **Tool calling**: `test_bedrock_realtime_tools.py`
## Requirements
```bash
pip install litellm websockets pyaudio
```
## AWS Configuration
Ensure your AWS credentials are configured:
```bash
export AWS_ACCESS_KEY_ID=your_access_key
export AWS_SECRET_ACCESS_KEY=your_secret_key
export AWS_REGION_NAME=us-east-1
```
Or use AWS CLI configuration:
```bash
aws configure
```
## Troubleshooting
### Connection Issues
- Ensure LiteLLM proxy is running on the correct port
- Verify AWS credentials are properly configured
- Check that the Bedrock model is available in your region
### Audio Issues
- Verify PyAudio is properly installed
- Check microphone/speaker permissions
- Ensure correct sample rates (16kHz input, 24kHz output)
### Tool Calling Issues
- Ensure tools are properly defined in session.update
- Verify tool results are sent back with correct call_id
- Check that response.create is sent after tool result
## Related Resources
- [OpenAI Realtime API Documentation](https://platform.openai.com/docs/guides/realtime)
- [Amazon Bedrock Nova Sonic Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/nova-sonic.html)
- [LiteLLM Realtime API Documentation](/docs/realtime)

51
docs/my-website/docs/providers/gemini.md
View file

@ -1840,6 +1840,57 @@ content = response.get('choices', [{}])[0].get('message', {}).get('content')
print(content)
```
## gemini-robotics-er-1.5-preview Usage
```python
from litellm import api_base
from openai import OpenAI
import os
import base64
client = OpenAI(base_url="http://0.0.0.0:4000", api_key="sk-12345")
base64_image = base64.b64encode(open("closeup-object-on-table-many-260nw-1216144471.webp", "rb").read()).decode()
import json
import re
tools = [{"codeExecution": {}}]
response = client.chat.completions.create(
model="gemini/gemini-robotics-er-1.5-preview",
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "Point to no more than 10 items in the image. The label returned should be an identifying name for the object detected. The answer should follow the json format: [{\"point\": [y, x], \"label\": <label1>}, ...]. The points are in [y, x] format normalized to 0-1000."
},
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}
}
]
}
],
tools=tools
)
# Extract JSON from markdown code block if present
content = response.choices[0].message.content
# Look for triple-backtick JSON block
match = re.search(r'```json\s*(.*?)\s*```', content, re.DOTALL)
if match:
json_str = match.group(1)
else:
json_str = content
try:
data = json.loads(json_str)
print(json.dumps(data, indent=2))
except Exception as e:
print("Error parsing response as JSON:", e)
print("Response content:", content)
```
## Usage - PDF / Videos / etc. Files
### Inline Data (e.g. audio stream)

35
docs/my-website/docs/providers/github_copilot.md
View file

@ -35,11 +35,10 @@ from litellm import completion
response = completion(
model="github_copilot/gpt-4",
messages=[{"role": "user", "content": "Write a Python function to calculate fibonacci numbers"}],
extra_headers={
"editor-version": "vscode/1.85.1",
"Copilot-Integration-Id": "vscode-chat"
}
messages=[
{"role": "system", "content": "You are a helpful coding assistant"},
{"role": "user", "content": "Write a Python function to calculate fibonacci numbers"}
]
)
print(response)
```
@ -50,11 +49,7 @@ from litellm import completion
stream = completion(
model="github_copilot/gpt-4",
messages=[{"role": "user", "content": "Explain async/await in Python"}],
stream=True,
extra_headers={
"editor-version": "vscode/1.85.1",
"Copilot-Integration-Id": "vscode-chat"
}
stream=True
)
for chunk in stream:
@ -134,11 +129,7 @@ client = OpenAI(
# Non-streaming response
response = client.chat.completions.create(
model="github_copilot/gpt-4",
messages=[{"role": "user", "content": "How do I optimize this SQL query?"}],
extra_headers={
"editor-version": "vscode/1.85.1",
"Copilot-Integration-Id": "vscode-chat"
}
messages=[{"role": "user", "content": "How do I optimize this SQL query?"}]
)
print(response.choices[0].message.content)
@ -156,11 +147,7 @@ response = litellm.completion(
model="litellm_proxy/github_copilot/gpt-4",
messages=[{"role": "user", "content": "Review this code for bugs"}],
api_base="http://localhost:4000",
api_key="your-proxy-api-key",
extra_headers={
"editor-version": "vscode/1.85.1",
"Copilot-Integration-Id": "vscode-chat"
}
api_key="your-proxy-api-key"
)
print(response.choices[0].message.content)
@ -174,8 +161,6 @@ print(response.choices[0].message.content)
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-H "editor-version: vscode/1.85.1" \
-H "Copilot-Integration-Id: vscode-chat" \
-d '{
"model": "github_copilot/gpt-4",
"messages": [{"role": "user", "content": "Explain this error message"}]
@ -211,9 +196,11 @@ export GITHUB_COPILOT_API_KEY_FILE="api-key.json"
### Headers
GitHub Copilot supports various editor-specific headers:
LiteLLM automatically injects the required GitHub Copilot headers (simulating VSCode). You don't need to specify them manually.
```python showLineNumbers title="Common Headers"
If you want to override the defaults (e.g., to simulate a different editor), you can use `extra_headers`:
```python showLineNumbers title="Custom Headers (Optional)"
extra_headers = {
"editor-version": "vscode/1.85.1", # Editor version
"editor-plugin-version": "copilot/1.155.0", # Plugin version

92
docs/my-website/docs/providers/sarvam.md Normal file
View file

@ -0,0 +1,92 @@
# Sarvam.ai
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM supports all the text models from [Sarvam ai](https://docs.sarvam.ai/api-reference-docs/chat/chat-completions)
## Usage
```python
import os
from litellm import completion
# Set your Sarvam API key
os.environ["SARVAM_API_KEY"] = ""
messages = [{"role": "user", "content": "Hello"}]
response = completion(
model="sarvam/sarvam-m",
messages=messages,
)
print(response)
```
## Usage with LiteLLM Proxy Server
Here's how to call a Sarvam.ai model with the LiteLLM Proxy Server
1. **Modify the `config.yaml`:**
```yaml
model_list:
- model_name: my-model
litellm_params:
model: sarvam/<your-model-name> # add sarvam/ prefix to route as Sarvam provider
api_key: api-key # api key to send your model
```
2. **Start the proxy:**
```bash
$ litellm --config /path/to/config.yaml
```
3. **Send a request to LiteLLM Proxy Server:**
<Tabs>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
base_url="http://0.0.0.0:4000" # litellm-proxy-base url
)
response = client.chat.completions.create(
model="my-model",
messages=[
{
"role": "user",
"content": "what llm are you"
}
],
)
print(response)
```
</TabItem>
<TabItem value="curl" label="curl">
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "my-model",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>

36
docs/my-website/docs/providers/vercel_ai_gateway.md
View file

@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem';
| Provider Route on LiteLLM | `vercel_ai_gateway/` |
| Link to Provider Doc | [Vercel AI Gateway Documentation ↗](https://vercel.com/docs/ai-gateway) |
| Base URL | `https://ai-gateway.vercel.sh/v1` |
| Supported Operations | `/chat/completions`, `/models` |
| Supported Operations | `/chat/completions`, `/embeddings`, `/models` |
<br />
<br />
@ -73,7 +73,7 @@ messages = [{"content": "Hello, how are you?", "role": "user"}]
# Vercel AI Gateway call with streaming
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
model="vercel_ai_gateway/openai/gpt-4o",
messages=messages,
stream=True
)
@ -82,6 +82,33 @@ for chunk in response:
print(chunk)
```
### Embeddings
```python showLineNumbers title="Vercel AI Gateway Embeddings"
import os
from litellm import embedding
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
# Vercel AI Gateway embedding call
response = embedding(
model="vercel_ai_gateway/openai/text-embedding-3-small",
input="Hello world"
)
print(response.data[0]["embedding"][:5]) # Print first 5 dimensions
```
You can also specify the `dimensions` parameter:
```python showLineNumbers title="Vercel AI Gateway Embeddings with Dimensions"
response = embedding(
model="vercel_ai_gateway/openai/text-embedding-3-small",
input=["Hello world", "Goodbye world"],
dimensions=768
)
```
## Usage - LiteLLM Proxy
Add the following to your LiteLLM Proxy configuration file:
@ -97,6 +124,11 @@ model_list:
litellm_params:
model: vercel_ai_gateway/anthropic/claude-4-sonnet
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
- model_name: text-embedding-3-small-gateway
litellm_params:
model: vercel_ai_gateway/openai/text-embedding-3-small
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
```
Start your LiteLLM Proxy server:

5
docs/my-website/docs/providers/vertex_speech.md
View file

@ -312,6 +312,7 @@ Gemini models with audio output capabilities using the chat completions API.
- Only supports `pcm16` audio format
- Streaming not yet supported
- Must set `modalities: ["audio"]`
- When using via LiteLLM Proxy, must include `"allowed_openai_params": ["audio", "modalities"]` in the request body to enable audio parameters
:::
### Quick Start
@ -372,7 +373,8 @@ curl http://0.0.0.0:4000/v1/chat/completions \
"model": "gemini-tts",
"messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
"modalities": ["audio"],
"audio": {"voice": "Kore", "format": "pcm16"}
"audio": {"voice": "Kore", "format": "pcm16"},
"allowed_openai_params": ["audio", "modalities"]
}'
```
@ -389,6 +391,7 @@ response = client.chat.completions.create(
messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
modalities=["audio"],
audio={"voice": "Kore", "format": "pcm16"},
extra_body={"allowed_openai_params": ["audio", "modalities"]}
)
print(response)
```

308
docs/my-website/docs/providers/xai_realtime.md Normal file
View file

@ -0,0 +1,308 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# xAI Voice Agent (Realtime API)
xAI's Grok Voice Agent provides real-time voice conversation capabilities through WebSocket connections, enabling natural bidirectional audio interactions.
| Feature | Description | Comments |
| --- | --- | --- |
| LiteLLM AI Gateway | ✅ | |
| LiteLLM Python SDK | ✅ | Full support via `litellm.realtime()` |
## Quick Start
### Supported Model
| Model | Context | Features |
|-------|---------|----------|
| `xai/grok-4-1-fast-non-reasoning` | 2M tokens | Voice conversation, Function calling, Vision, Audio, Web search, Caching |
**Note:** xAI Realtime API uses the non-reasoning variant for optimal real-time performance.
## Python SDK Usage
### Basic Realtime Connection
```python
import asyncio
from litellm import realtime
async def test_xai_realtime():
"""
Test xAI Grok Voice Agent via LiteLLM SDK
"""
# Initialize realtime connection
ws = await realtime(
model="xai/grok-4-1-fast-non-reasoning",
api_key="your-xai-api-key", # or set XAI_API_KEY env var
)
# Connection established, xAI sends "conversation.created" event
print("Connected to xAI Grok Voice Agent")
# Send a message
await ws.send_text(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "Hello! How are you?"
}]
}
}))
# Request a response
await ws.send_text(json.dumps({
"type": "response.create"
}))
# Listen for responses
async for message in ws:
data = json.loads(message)
print(f"Received: {data['type']}")
if data['type'] == 'response.done':
break
await ws.close()
# Run the async function
asyncio.run(test_xai_realtime())
```
### With Audio Input/Output
```python
import asyncio
import json
from litellm import realtime
async def xai_voice_conversation():
"""
Voice conversation with xAI Grok Voice Agent
"""
ws = await realtime(
model="xai/grok-4-1-fast-non-reasoning",
api_key="your-xai-api-key",
)
# Send audio data (base64 encoded PCM16 24kHz)
await ws.send_text(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{
"type": "input_audio",
"audio": "base64_encoded_audio_data_here"
}]
}
}))
# Request response with audio
await ws.send_text(json.dumps({
"type": "response.create",
"response": {
"modalities": ["text", "audio"],
"instructions": "Please respond in a friendly tone."
}
}))
# Process streaming audio response
async for message in ws:
data = json.loads(message)
if data['type'] == 'response.audio.delta':
# Handle audio chunks
audio_chunk = data['delta']
# Process audio_chunk (play it, save it, etc.)
elif data['type'] == 'response.done':
break
await ws.close()
asyncio.run(xai_voice_conversation())
```
## LiteLLM Proxy (AI Gateway) Usage
Load balance across multiple xAI deployments or combine with other providers.
### 1. Add Model to Config
```yaml
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-4-1-fast-non-reasoning
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
# Optional: Add fallback to OpenAI
- model_name: grok-voice-agent
litellm_params:
model: openai/gpt-4o-realtime-preview-2024-10-01
api_key: os.environ/OPENAI_API_KEY
model_info:
mode: realtime
```
### 2. Start Proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Test Connection
#### Python Client
```python
import asyncio
import websockets
import json
async def test_proxy():
url = "ws://0.0.0.0:4000/v1/realtime?model=grok-voice-agent"
async with websockets.connect(
url,
extra_headers={
"Authorization": "Bearer sk-1234", # Your LiteLLM proxy key
"OpenAI-Beta": "realtime=v1"
}
) as ws:
# Wait for conversation.created event from xAI
message = await ws.recv()
print(f"Connected: {message}")
# Send a message
await ws.send(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "Hello from LiteLLM proxy!"
}]
}
}))
# Request response
await ws.send(json.dumps({
"type": "response.create"
}))
# Listen for response
async for message in ws:
data = json.loads(message)
print(f"Event: {data['type']}")
if data['type'] == 'response.done':
break
asyncio.run(test_proxy())
```
#### Node.js Client
```javascript
// test.js - Run with: node test.js
const WebSocket = require("ws");
const url = "ws://0.0.0.0:4000/v1/realtime?model=grok-voice-agent";
const ws = new WebSocket(url, {
headers: {
"Authorization": "Bearer sk-1234",
"OpenAI-Beta": "realtime=v1",
},
});
ws.on("open", function open() {
console.log("Connected to xAI via LiteLLM proxy");
// Send a message
ws.send(JSON.stringify({
type: "conversation.item.create",
item: {
type: "message",
role: "user",
content: [{
type: "input_text",
text: "What's the weather like?"
}]
}
}));
// Request response
ws.send(JSON.stringify({
type: "response.create",
response: {
modalities: ["text"],
instructions: "Please assist the user."
}
}));
});
ws.on("message", function incoming(message) {
const data = JSON.parse(message.toString());
console.log(`Event: ${data.type}`);
if (data.type === 'response.done') {
ws.close();
}
});
ws.on("error", function handleError(error) {
console.error("Error: ", error);
});
```
## Key Differences from OpenAI
xAI's Grok Voice Agent has some differences from OpenAI's Realtime API:
| Feature | xAI | OpenAI | LiteLLM Handling |
|---------|-----|--------|------------------|
| Initial Event | `conversation.created` | `session.created` | ⚠️ Passed through as-is |
| WebSocket URL | `wss://api.x.ai/v1/realtime` | `wss://api.openai.com/v1/realtime` | ✅ Auto-configured |
| Model | `grok-4-1-fast-non-reasoning` | `gpt-4o-realtime-preview` | ✅ Via model prefix |
| Audio Format | PCM16 24kHz mono | PCM16 24kHz mono | ✅ Compatible |
| Context Window | 2M tokens | 128K tokens | N/A |
**What LiteLLM Handles:**
- ✅ Automatic URL routing to correct provider
- ✅ Authentication headers (no `OpenAI-Beta` header for xAI)
- ✅ WebSocket connection management
- ✅ All other event types are compatible
**What You Need to Handle:**
- ⚠️ Initial event type difference (`conversation.created` vs `session.created`)
**Tip:** Make your client compatible with both event types:
```python
# Handle both providers
if event['type'] in ['session.created', 'conversation.created']:
print("Connection established")
```
## Related Documentation
- [xAI Chat/Text Models](/docs/providers/xai)
- [LiteLLM Realtime API Overview](/docs/realtime)
- [xAI Official Documentation](https://docs.x.ai/docs)
## Support
For issues or questions:
- [LiteLLM GitHub Issues](https://github.com/BerriAI/litellm/issues)
- [xAI Documentation](https://docs.x.ai/docs)

41
docs/my-website/docs/proxy/call_hooks.md
View file

@ -19,6 +19,7 @@ import Image from '@theme/IdealImage';
| `async_post_call_success_hook` | Modify outgoing response (non-streaming) | After successful LLM API call, for non-streaming responses |
| `async_post_call_failure_hook` | Transform error responses sent to clients | After failed LLM API call |
| `async_post_call_streaming_hook` | Modify outgoing response (streaming) | After successful LLM API call, for streaming responses |
| `async_post_call_response_headers_hook` | Inject custom HTTP response headers | After LLM API call (both success and failure) |
See a complete example with our [parallel request rate limiter](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/parallel_request_limiter.py)
@ -115,6 +116,18 @@ class MyCustomHandler(CustomLogger): # https://docs.litellm.ai/docs/observabilit
async for item in response:
yield item
async def async_post_call_response_headers_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
request_headers: Optional[Dict[str, str]] = None,
) -> Optional[Dict[str, str]]:
"""
Inject custom headers into HTTP response (runs for both success and failure).
"""
return {"x-custom-header": "custom-value"}
proxy_handler_instance = MyCustomHandler()
```
@ -389,3 +402,31 @@ proxy_handler_instance = MyErrorTransformer()
```
**Result:** Clients receive `"Your prompt is too long..."` instead of `"ContextWindowExceededError: Prompt exceeds context window"`.
## Advanced - Inject Custom HTTP Response Headers
Use `async_post_call_response_headers_hook` to inject custom HTTP headers into responses. This hook runs for **both successful and failed** LLM API calls.
```python
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy.proxy_server import UserAPIKeyAuth
from typing import Any, Dict, Optional
class CustomHeaderLogger(CustomLogger):
def __init__(self):
super().__init__()
async def async_post_call_response_headers_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
request_headers: Optional[Dict[str, str]] = None,
) -> Optional[Dict[str, str]]:
"""
Inject custom headers into all responses (success and failure).
"""
return {"x-custom-header": "custom-value"}
proxy_handler_instance = CustomHeaderLogger()
```

31
docs/my-website/docs/proxy/cli_sso.md
View file

@ -28,6 +28,37 @@ EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
:::
### Configuration
#### JWT Token Expiration
By default, CLI authentication tokens expire after **24 hours**. You can customize this expiration time by setting the `LITELLM_CLI_JWT_EXPIRATION_HOURS` environment variable when starting your LiteLLM Proxy:
```bash
# Set CLI JWT tokens to expire after 48 hours
export LITELLM_CLI_JWT_EXPIRATION_HOURS=48
export EXPERIMENTAL_UI_LOGIN="True"
litellm --config config.yaml
```
Or in a single command:
```bash
LITELLM_CLI_JWT_EXPIRATION_HOURS=48 EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
```
**Examples:**
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=12` - Tokens expire after 12 hours
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=168` - Tokens expire after 7 days (168 hours)
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=720` - Tokens expire after 30 days (720 hours)
:::tip
You can check your current token's age and expiration status using:
```bash
litellm-proxy whoami
```
:::
### Steps
1. **Install the CLI**

30
docs/my-website/docs/proxy/config_settings.md
View file

@ -94,7 +94,7 @@ litellm_settings:
# /chat/completions, /completions, /embeddings, /audio/transcriptions
mode: default_off # if default_off, you need to opt in to caching on a per call basis
ttl: 600 # ttl for caching
disable_copilot_system_to_assistant: False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
disable_copilot_system_to_assistant: False # DEPRECATED - GitHub Copilot API supports system prompts.
callback_settings:
otel:
@ -197,7 +197,7 @@ router_settings:
| disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. |
| disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
| disable_copilot_system_to_assistant | boolean | If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. Useful for tools (like Claude Code) that send system messages, which Copilot does not support. |
| disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
### general_settings - Reference
@ -321,6 +321,7 @@ router_settings:
| redis_host | string | The host address for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them** |
| redis_password | string | The password for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them** |
| redis_port | string | The port number for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them**|
| redis_db | int | The database number for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them**|
| enable_pre_call_check | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) |
| content_policy_fallbacks | array of objects | Specifies fallback models for content policy violations. [More information here](reliability) |
| fallbacks | array of objects | Specifies fallback models for all types of errors. [More information here](reliability) |
@ -452,6 +453,8 @@ router_settings:
| BERRISPEND_ACCOUNT_ID | Account ID for BerriSpend service
| BRAINTRUST_API_KEY | API key for Braintrust integration
| BRAINTRUST_API_BASE | Base URL for Braintrust API. Default is https://api.braintrustdata.com/v1
| BRAINTRUST_MOCK | Enable mock mode for Braintrust integration testing. When set to true, intercepts Braintrust API calls and returns mock responses without making actual network calls. Default is false
| BRAINTRUST_MOCK_LATENCY_MS | Mock latency in milliseconds for Braintrust API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| CACHED_STREAMING_CHUNK_DELAY | Delay in seconds for cached streaming chunks. Default is 0.02
| CHATGPT_API_BASE | Base URL for ChatGPT API. Default is https://chatgpt.com/backend-api/codex
| CHATGPT_AUTH_FILE | Filename for ChatGPT authentication data. Default is "auth.json"
@ -462,6 +465,7 @@ router_settings:
| CHATGPT_USER_AGENT_SUFFIX | Suffix to append to the ChatGPT user agent string
| CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI
| CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
| CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours. Can also be set via LITELLM_CLI_JWT_EXPIRATION_HOURS
| CLOUDZERO_API_KEY | CloudZero API key for authentication
| CLOUDZERO_CONNECTION_ID | CloudZero connection ID for data submission
| CLOUDZERO_EXPORT_INTERVAL_MINUTES | Interval in minutes for CloudZero data export operations
@ -504,12 +508,15 @@ router_settings:
| DD_AGENT_HOST | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API
| DD_AGENT_PORT | Port of DataDog agent for log intake. Default is 10518
| DD_API_KEY | API key for Datadog integration
| DD_APP_KEY | Application key for Datadog Cost Management integration. Required along with DD_API_KEY for cost metrics
| DD_SITE | Site URL for Datadog (e.g., datadoghq.com)
| DD_SOURCE | Source identifier for Datadog logs
| DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE | Resource name for Datadog tracing of streaming chunk yields. Default is "streaming.chunk.yield"
| DD_ENV | Environment identifier for Datadog logs. Only supported for `datadog_llm_observability` callback
| DD_SERVICE | Service identifier for Datadog logs. Defaults to "litellm-server"
| DD_VERSION | Version identifier for Datadog logs. Defaults to "unknown"
| DATADOG_MOCK | Enable mock mode for Datadog integration testing. When set to true, intercepts Datadog API calls and returns mock responses without making actual network calls. Default is false
| DATADOG_MOCK_LATENCY_MS | Mock latency in milliseconds for Datadog API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| DEBUG_OTEL | Enable debug mode for OpenTelemetry
| DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3
| DEFAULT_A2A_AGENT_TIMEOUT | Default timeout in seconds for A2A (Agent-to-Agent) protocol requests. Default is 6000
@ -538,6 +545,9 @@ router_settings:
| DEFAULT_MAX_TOKENS | Default maximum tokens for LLM calls. Default is 4096
| DEFAULT_MAX_TOKENS_FOR_TRITON | Default maximum tokens for Triton models. Default is 2000
| DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE | Default maximum size for redis batch cache. Default is 1000
| DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL | Default embedding model for MCP semantic tool filtering. Default is "text-embedding-3-small"
| DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD | Default similarity threshold for MCP semantic tool filtering. Default is 0.3
| DEFAULT_MCP_SEMANTIC_FILTER_TOP_K | Default number of top results to return for MCP semantic tool filtering. Default is 10
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
@ -638,6 +648,10 @@ router_settings:
| GENERIC_USERINFO_ENDPOINT | Endpoint to fetch user information in generic OAuth
| GENERIC_LOGGER_ENDPOINT | Endpoint URL for the Generic Logger callback to send logs to
| GENERIC_LOGGER_HEADERS | JSON string of headers to include in Generic Logger callback requests
| GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE | Default LiteLLM role to assign when no role mapping matches in generic SSO. Used with GENERIC_ROLE_MAPPINGS_ROLES
| GENERIC_ROLE_MAPPINGS_GROUP_CLAIM | The claim/attribute name in the SSO token that contains the user's groups. Used for role mapping
| GENERIC_ROLE_MAPPINGS_ROLES | Python dict string mapping LiteLLM roles to SSO group names. Example: `{"proxy_admin": ["admin-group"], "internal_user": ["users"]}`
| GENERIC_USER_ROLE_MAPPINGS | Alternative to GENERIC_ROLE_MAPPINGS_ROLES for configuring user role mappings from SSO
| GEMINI_API_BASE | Base URL for Gemini API. Default is https://generativelanguage.googleapis.com
| GALILEO_BASE_URL | Base URL for Galileo platform
| GALILEO_PASSWORD | Password for Galileo authentication
@ -674,6 +688,8 @@ router_settings:
| HCP_VAULT_CERT_ROLE | Role for [Hashicorp Vault Secret Manager Auth](../secret.md#hashicorp-vault)
| HELICONE_API_KEY | API key for Helicone service
| HELICONE_API_BASE | Base URL for Helicone service, defaults to `https://api.helicone.ai`
| HELICONE_MOCK | Enable mock mode for Helicone integration testing. When set to true, intercepts Helicone API calls and returns mock responses without making actual network calls. Default is false
| HELICONE_MOCK_LATENCY_MS | Mock latency in milliseconds for Helicone API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| HOSTNAME | Hostname for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog)
| HOURS_IN_A_DAY | Hours in a day for calculation purposes. Default is 24
| HIDDENLAYER_API_BASE | Base URL for HiddenLayer API. Defaults to `https://api.hiddenlayer.ai`
@ -712,6 +728,8 @@ router_settings:
| LANGSMITH_PROJECT | Project name for Langsmith integration
| LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging
| LANGSMITH_TENANT_ID | Tenant ID for Langsmith multi-tenant deployments
| LANGSMITH_MOCK | Enable mock mode for Langsmith integration testing. When set to true, intercepts Langsmith API calls and returns mock responses without making actual network calls. Default is false
| LANGSMITH_MOCK_LATENCY_MS | Mock latency in milliseconds for Langsmith API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| LANGTRACE_API_KEY | API key for Langtrace service
| LASSO_API_BASE | Base URL for Lasso API
| LASSO_API_KEY | API key for Lasso service
@ -723,8 +741,10 @@ router_settings:
| LITERAL_API_URL | API URL for Literal service
| LITERAL_BATCH_SIZE | Batch size for Literal operations
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
| LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
| LITELLM_DD_LLM_OBS_PORT | Port for Datadog LLM Observability agent. Default is 8126
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
| LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
@ -785,6 +805,7 @@ router_settings:
| MAXIMUM_TRACEBACK_LINES_TO_LOG | Maximum number of lines to log in traceback in LiteLLM Logs UI. Default is 100
| MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0
| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 50. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
| MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH | Maximum header length for MCP semantic filter tools. Default is 150
| MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001
| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
| MISTRAL_API_BASE | Base URL for Mistral API. Default is https://api.mistral.ai
@ -820,6 +841,7 @@ router_settings:
| OPENMETER_EVENT_TYPE | Type of events sent to OpenMeter
| ONYX_API_BASE | Base URL for Onyx Security AI Guard service (defaults to https://ai-guard.onyx.security)
| ONYX_API_KEY | API key for Onyx Security AI Guard service
| ONYX_TIMEOUT | Timeout in seconds for Onyx Guard server requests. Default is 10
| OTEL_ENDPOINT | OpenTelemetry endpoint for traces
| OTEL_EXPORTER_OTLP_ENDPOINT | OpenTelemetry endpoint for traces
| OTEL_ENVIRONMENT_NAME | Environment name for OpenTelemetry
@ -843,6 +865,8 @@ router_settings:
| POD_NAME | Pod name for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog) as `POD_NAME`
| POSTHOG_API_KEY | API key for PostHog analytics integration
| POSTHOG_API_URL | Base URL for PostHog API (defaults to https://us.i.posthog.com)
| POSTHOG_MOCK | Enable mock mode for PostHog integration testing. When set to true, intercepts PostHog API calls and returns mock responses without making actual network calls. Default is false
| POSTHOG_MOCK_LATENCY_MS | Mock latency in milliseconds for PostHog API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| PREDIBASE_API_BASE | Base URL for Predibase API
| PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
| PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service
@ -880,6 +904,8 @@ router_settings:
| ROUTER_MAX_FALLBACKS | Maximum number of fallbacks for router. Default is 5
| RUNWAYML_DEFAULT_API_VERSION | Default API version for RunwayML service. Default is "2024-11-06"
| RUNWAYML_POLLING_TIMEOUT | Timeout in seconds for RunwayML image generation polling. Default is 600 (10 minutes)
| S3_VECTORS_DEFAULT_DIMENSION | Default vector dimension for S3 Vectors RAG ingestion. Default is 1024
| S3_VECTORS_DEFAULT_DISTANCE_METRIC | Default distance metric for S3 Vectors RAG ingestion. Options: "cosine", "euclidean". Default is "cosine"
| SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours)
| SEPARATE_HEALTH_APP | If set to '1', runs health endpoints on a separate ASGI app and port. Default: '0'.
| SEPARATE_HEALTH_PORT | Port for the separate health endpoints app. Only used if SEPARATE_HEALTH_APP=1. Default: 4001.

46
docs/my-website/docs/proxy/custom_pricing.md
View file

@ -9,6 +9,7 @@ LiteLLM provides flexible cost tracking and pricing customization for all LLM pr
- **Custom Pricing** - Override default model costs or set pricing for custom models
- **Cost Per Token** - Track costs based on input/output tokens (most common)
- **Cost Per Second** - Track costs based on runtime (e.g., Sagemaker)
- **Zero-Cost Models** - Bypass budget checks for free/on-premises models by setting costs to 0
- **[Provider Discounts](./provider_discounts.md)** - Apply percentage-based discounts to specific providers
- **[Provider Margins](./provider_margins.md)** - Add fees/margins to LLM costs for internal billing
- **Base Model Mapping** - Ensure accurate cost tracking for Azure deployments
@ -106,6 +107,51 @@ There are other keys you can use to specify costs for different scenarios and mo
These keys evolve based on how new models handle multimodality. The latest version can be found at [https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
## Zero-Cost Models (Bypass Budget Checks)
**Use Case**: You have on-premises or free models that should be accessible even when users exceed their budget limits.
**Solution** ✅: Set both `input_cost_per_token` and `output_cost_per_token` to `0` (explicitly) to bypass all budget checks for that model.
:::info
When a model is configured with zero cost, LiteLLM will automatically skip ALL budget checks (user, team, team member, end-user, organization, and global proxy budget) for requests to that model.
**Important**: Both costs must be **explicitly set to 0**. If costs are `null` or undefined, the model will be treated as having cost and budget checks will apply.
:::
### Configuration Example
```yaml
model_list:
# On-premises model - free to use
- model_name: on-prem-llama
litellm_params:
model: ollama/llama3
api_base: http://localhost:11434
model_info:
input_cost_per_token: 0 # 👈 Explicitly set to 0
output_cost_per_token: 0 # 👈 Explicitly set to 0
# Paid cloud model - budget checks apply
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
# No model_info - uses default pricing from cost map
```
### Behavior
With the above configuration:
- **User over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
- **Team over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
- **End-user over budget** → Can still use `on-prem-llama` ✅, but blocked from `gpt-4` ❌
This ensures your free/on-premises models remain accessible regardless of budget constraints, while paid models are still properly governed.
## Set 'base_model' for Cost Tracking (e.g. Azure deployments)
**Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking

10
docs/my-website/docs/proxy/embedding.md
View file

@ -6,6 +6,16 @@ import TabItem from '@theme/TabItem';
See supported Embedding Providers & Models [here](https://docs.litellm.ai/docs/embedding/supported_embedding)
## Supported Input Formats
The `/v1/embeddings` endpoint follows the [OpenAI embeddings API specification](https://platform.openai.com/docs/api-reference/embeddings/create). The following input formats are supported:
| Format | Example |
|--------|---------|
| String | `"input": "Hello"` |
| Array of strings | `"input": ["Hello", "World"]` |
| Array of tokens (integers) | `"input": [1234, 5678, 9012]` |
| Array of token arrays | `"input": [[1234, 5678], [9012, 3456]]` |
## Quick start
Here's how to route between GPT-J embedding (sagemaker endpoint), Amazon Titan embedding (Bedrock) and Azure OpenAI embedding on the proxy server:

278
docs/my-website/docs/proxy/guardrails/custom_code_guardrail.md Normal file
View file

@ -0,0 +1,278 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Custom Code Guardrail
Write custom guardrail logic using Python-like code that runs in a sandboxed environment.
## Quick Start
### 1. Define the guardrail in config
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: block-ssn
litellm_params:
guardrail: custom_code
mode: pre_call
custom_code: |
def apply_guardrail(inputs, request_data, input_type):
for text in inputs["texts"]:
if regex_match(text, r"\d{3}-\d{2}-\d{4}"):
return block("SSN detected")
return allow()
```
### 2. Start proxy
```bash
litellm --config config.yaml
```
### 3. Test
```bash
curl -X POST http://localhost:4000/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "My SSN is 123-45-6789"}],
"guardrails": ["block-ssn"]
}'
```
## Configuration
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `guardrail` | string | ✅ | Must be `custom_code` |
| `mode` | string | ✅ | When to run: `pre_call`, `post_call`, `during_call` |
| `custom_code` | string | ✅ | Python-like code with `apply_guardrail` function |
| `default_on` | bool | ❌ | Run on all requests (default: `false`) |
## Writing Custom Code
### Function Signature
Your code must define an `apply_guardrail` function:
```python
def apply_guardrail(inputs, request_data, input_type):
# inputs: see table below
# request_data: {"model": "...", "user_id": "...", "team_id": "...", "metadata": {...}}
# input_type: "request" or "response"
return allow() # or block() or modify()
```
### `inputs` Parameter
| Field | Type | Description |
|-------|------|-------------|
| `texts` | `List[str]` | Extracted text from the request/response |
| `images` | `List[str]` | Extracted images (for image guardrails) |
| `tools` | `List[dict]` | Tools sent to the LLM |
| `tool_calls` | `List[dict]` | Tool calls returned from the LLM |
| `structured_messages` | `List[dict]` | Full messages with role info (system/user/assistant) |
| `model` | `str` | The model being used |
### `request_data` Parameter
| Field | Type | Description |
|-------|------|-------------|
| `model` | `str` | Model name |
| `user_id` | `str` | User ID from API key |
| `team_id` | `str` | Team ID from API key |
| `end_user_id` | `str` | End user ID |
| `metadata` | `dict` | Request metadata |
### Return Values
| Function | Description |
|----------|-------------|
| `allow()` | Let request/response through |
| `block(reason)` | Reject with message |
| `modify(texts=[], images=[], tool_calls=[])` | Transform content |
## Built-in Primitives
### Regex
| Function | Description |
|----------|-------------|
| `regex_match(text, pattern)` | Returns `True` if pattern found |
| `regex_replace(text, pattern, replacement)` | Replace all matches |
| `regex_find_all(text, pattern)` | Return list of matches |
### JSON
| Function | Description |
|----------|-------------|
| `json_parse(text)` | Parse JSON string, returns `None` on error |
| `json_stringify(obj)` | Convert to JSON string |
| `json_schema_valid(obj, schema)` | Validate against JSON schema |
### URL
| Function | Description |
|----------|-------------|
| `extract_urls(text)` | Extract all URLs from text |
| `is_valid_url(url)` | Check if URL is valid |
| `all_urls_valid(text)` | Check all URLs in text are valid |
### Code Detection
| Function | Description |
|----------|-------------|
| `detect_code(text)` | Returns `True` if code detected |
| `detect_code_languages(text)` | Returns list of detected languages |
| `contains_code_language(text, ["sql", "python"])` | Check for specific languages |
### Text Utilities
| Function | Description |
|----------|-------------|
| `contains(text, substring)` | Check if substring exists |
| `contains_any(text, [substr1, substr2])` | Check if any substring exists |
| `word_count(text)` | Count words |
| `char_count(text)` | Count characters |
| `lower(text)` / `upper(text)` / `trim(text)` | String transforms |
## Examples
### Block PII (SSN)
```python
def apply_guardrail(inputs, request_data, input_type):
for text in inputs["texts"]:
if regex_match(text, r"\d{3}-\d{2}-\d{4}"):
return block("SSN detected")
return allow()
```
### Redact Email Addresses
```python
def apply_guardrail(inputs, request_data, input_type):
pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"
modified = []
for text in inputs["texts"]:
modified.append(regex_replace(text, pattern, "[EMAIL REDACTED]"))
return modify(texts=modified)
```
### Block SQL Injection
```python
def apply_guardrail(inputs, request_data, input_type):
if input_type != "request":
return allow()
for text in inputs["texts"]:
if contains_code_language(text, ["sql"]):
return block("SQL code not allowed")
return allow()
```
### Validate JSON Response
```python
def apply_guardrail(inputs, request_data, input_type):
if input_type != "response":
return allow()
schema = {
"type": "object",
"required": ["name", "value"]
}
for text in inputs["texts"]:
obj = json_parse(text)
if obj is None:
return block("Invalid JSON response")
if not json_schema_valid(obj, schema):
return block("Response missing required fields")
return allow()
```
### Check URLs in Response
```python
def apply_guardrail(inputs, request_data, input_type):
if input_type != "response":
return allow()
for text in inputs["texts"]:
if not all_urls_valid(text):
return block("Response contains invalid URLs")
return allow()
```
### Combine Multiple Checks
```python
def apply_guardrail(inputs, request_data, input_type):
modified = []
for text in inputs["texts"]:
# Redact SSN
text = regex_replace(text, r"\d{3}-\d{2}-\d{4}", "[SSN]")
# Redact credit cards
text = regex_replace(text, r"\d{16}", "[CARD]")
modified.append(text)
# Block SQL in requests
if input_type == "request":
for text in inputs["texts"]:
if contains_code_language(text, ["sql"]):
return block("SQL injection blocked")
return modify(texts=modified)
```
## Sandbox Restrictions
Custom code runs in a restricted environment:
- ❌ No `import` statements
- ❌ No file I/O
- ❌ No network access
- ❌ No `exec()` or `eval()`
- ✅ Only LiteLLM-provided primitives available
## Per-Request Usage
Enable guardrail per request:
```bash
curl -X POST http://localhost:4000/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
"guardrails": ["block-ssn"]
}'
```
## Default On
Run guardrail on all requests:
```yaml
litellm_settings:
guardrails:
- guardrail_name: block-ssn
litellm_params:
guardrail: custom_code
mode: pre_call
default_on: true
custom_code: |
def apply_guardrail(inputs, request_data, input_type):
...
```

197
docs/my-website/docs/proxy/guardrails/grayswan.md
View file

@ -13,20 +13,26 @@ Cygnal returns a `violation` score between `0` and `1` (higher means more likely
### 1. Obtain Credentials
1. Create a Gray Swan account and generate a Cygnal API key.
1. Log in to our Gray Swan platform and generate a Cygnal API key.
For existing customers, you should already have access to our [platform](https://platform.grayswan.ai).
For new users, please register at this [page](https://hubs.ly/Q03-sX1J0) and we are more than happy to give you an onboarding!
2. Configure environment variables for the LiteLLM proxy host:
```bash
export GRAYSWAN_API_KEY="your-grayswan-key"
export GRAYSWAN_API_BASE="https://api.grayswan.ai"
```
```bash
export GRAYSWAN_API_KEY="your-grayswan-key"
export GRAYSWAN_API_BASE="https://api.grayswan.ai"
```
### 2. Configure `config.yaml`
Add a guardrail entry that references the Gray Swan integration. Below is a balanced example that monitors both input and output but only blocks once the violation score reaches the configured threshold.
Add a guardrail entry that references the Gray Swan integration. Below is our recommmended settings.
```yaml
model_list:
model_list: # this part is a standard litellm configuration for reference
- model_name: openai/gpt-4.1-mini
litellm_params:
model: openai/gpt-4.1-mini
@ -40,13 +46,14 @@ guardrails:
api_key: os.environ/GRAYSWAN_API_KEY
api_base: os.environ/GRAYSWAN_API_BASE # optional
optional_params:
on_flagged_action: monitor # or "block"
on_flagged_action: passthrough # or "block" or "monitor"
violation_threshold: 0.5 # score >= threshold is flagged
reasoning_mode: hybrid # off | hybrid | thinking
categories:
safety: "Detect jailbreaks and policy violations"
policy_id: "your-cygnal-policy-id"
policy_id: "your-cygnal-policy-id" # Optional: Your Cygnal policy ID. Defaults to a content safety policy if empty.
streaming_end_of_stream_only: true # For streaming API, only send the assembled message to Cygnal (post_call only). Defaults to false.
default_on: true
guardrail_timeout: 30 # Defaults to 30 seconds. Change accordingly.
fail_open: true # Defaults to true; set to false to propagate guardrail errors.
general_settings:
master_key: "your-litellm-master-key"
@ -65,13 +72,13 @@ litellm --config config.yaml --port 4000
## Choosing Guardrail Modes
Gray Swan can run during `pre_call`, `during_call`, and `post_call` stages. Combine modes based on your latency and coverage requirements.
Gray Swan can run during `pre_call`, `during_call`, and `post_call` stages. Combine modes based on your latency and coverage requirements.
| Mode | When it Runs | Protects | Typical Use Case |
|--------------|-------------------|-----------------------|------------------|
| `pre_call` | Before LLM call | User input only | Block prompt injection before it reaches the model |
| `during_call`| Parallel to call | User input only | Low-latency monitoring without blocking |
| `post_call` | After response | Full conversation | Scan output for policy violations, leaked secrets, or IPI |
| `post_call` | After response | Model Outputs | Scan output for policy violations, leaked secrets, or IPI |
When using `during_call` with `on_flagged_action: block` or `on_flagged_action: passthrough`:
@ -81,87 +88,110 @@ When using `during_call` with `on_flagged_action: block` or `on_flagged_action:
- The guardrail exception prevents the response from reaching the user, but **does not cancel the running LLM task**
- This means you pay full LLM costs while returning an error/passthrough message to the user
**Recommendation:** For cost-sensitive applications, use `pre_call` and `post_call` instead of `during_call` for blocking or passthrough modes. Reserve `during_call` for `monitor` mode where you want low-latency logging without impacting the user experience.
**Recommendation:** Use `pre_call` and `post_call` instead of `during_call` for `passthrough` (or `block`) `on_flagged_action` (see our recommended configuration above). Reserve `during_call` for `monitor` mode ONLY when you want low-latency logging without impacting the user experience.
<Tabs>
<TabItem value="monitor" label="Monitor Only">
---
```yaml
guardrails:
- guardrail_name: "cygnal-monitor-only"
litellm_params:
guardrail: grayswan
mode: "during_call"
api_key: os.environ/GRAYSWAN_API_KEY
optional_params:
on_flagged_action: monitor
violation_threshold: 0.6
default_on: true
## Work with Claude Code
Follow the official litellm [guide](https://docs.litellm.ai/docs/tutorials/claude_responses_api) on setting up Claude Code with litellm, with the guardrail part mentioned above added to your litellm configuration. Cygnal natively supports coding agent policies defense. Define your own policy or use the provided coding policies on the platform. The example config we show above is also the recommended setup for Claude Code (with the `policy_id` replaced with an appropriate one).
---
## Per-request overrides via `extra_body`
You can override parts of the Gray Swan guardrail configuration on a per-request basis by passing `litellm_metadata.guardrails[*].grayswan.extra_body`.
`extra_body` is merged into the Cygnal request body and takes precedence over specific fields from `config.yaml`, which are `policy_id`, `violation_threshold`, and `reasoning_mode`.
If you include a `metadata` field inside `extra_body`, it is forwarded to the Cygnal API as-is under the request body's `metadata` field.
Example:
```bash
curl -X POST "http://0.0.0.0:4000/v1/messages?beta=true" \
-H "Authorization: Bearer token" \
-H "Content-Type: application/json" \
-d '{
"model": "openrouter/anthropic/claude-sonnet-4.5",
"messages": [{"role": "user", "content": "hello"}],
"litellm_metadata": {
"guardrails": [
{
"cygnal-monitor": {
"extra_body": {
"policy_id": "specific policy id you want to use",
"metadata": {
"user": "health-check"
}
}
}
}
]
}
}'
```
Best for visibility without blocking. Alerts are logged via LiteLLM’s standard logging callbacks.
OpenAI client:
</TabItem>
<TabItem value="block-input" label="Block Input">
```python
from openai import OpenAI
```yaml
guardrails:
- guardrail_name: "cygnal-block-input"
litellm_params:
guardrail: grayswan
mode: "pre_call"
api_key: os.environ/GRAYSWAN_API_KEY
optional_params:
on_flagged_action: block
violation_threshold: 0.4
categories:
pii: "Detect sensitive data"
default_on: true
client = OpenAI(api_key="anything", base_url="http://0.0.0.0:4000")
resp = client.responses.create(
model="openrouter/anthropic/claude-sonnet-4.5",
input="hello",
extra_body={
"litellm_metadata": {
"guardrails": [
{
"cygnal-monitor": {
"extra_body": {
"policy_id": "69038214e5cdb6befc5e991e",
"metadata": {"trace_id": "trace-123"},
}
}
}
]
}
},
)
```
Stops malicious or sensitive prompts before any tokens are generated.
Anthropic client:
</TabItem>
<TabItem value="full-coverage" label="Full Coverage">
```python
from anthropic import Anthropic
```yaml
guardrails:
- guardrail_name: "cygnal-full-coverage"
litellm_params:
guardrail: grayswan
mode: [pre_call, post_call]
api_key: os.environ/GRAYSWAN_API_KEY
optional_params:
on_flagged_action: block
violation_threshold: 0.5
reasoning_mode: thinking
policy_id: "policy-id-from-grayswan"
default_on: true
client = Anthropic(api_key="anything", base_url="http://0.0.0.0:4000")
resp = client.messages.create(
model="openrouter/anthropic/claude-sonnet-4.5",
max_tokens=256,
messages=[{"role": "user", "content": "hello"}],
extra_body={
"litellm_metadata": {
"guardrails": [
{
"cygnal-monitor": {
"extra_body": {
"policy_id": "69038214e5cdb6befc5e991e",
"metadata": {"trace_id": "trace-123"},
}
}
}
]
}
},
)
```
Provides the strongest enforcement by inspecting both prompts and responses.
Notes:
</TabItem>
<TabItem value="passthrough" label="Passthrough Mode">
```yaml
guardrails:
- guardrail_name: "cygnal-passthrough"
litellm_params:
guardrail: grayswan
mode: [pre_call, post_call]
api_key: os.environ/GRAYSWAN_API_KEY
optional_params:
on_flagged_action: passthrough
violation_threshold: 0.5
default_on: true
```
Allows requests to proceed without raising a 400 error when content is flagged. Instead of blocking, the model response content is replaced with a detailed violation message including violation score, violated rules, and detection flags (mutation, IPI). **Supported Response Formats:** OpenAI chat/text completions, Anthropic Messages API. Other response types (embeddings, images, etc.) will log a warning and return unchanged.
</TabItem>
</Tabs>
- The guardrail name (for example, `cygnal-monitor`) must match the `guardrail_name` in `config.yaml`.
- Per-request guardrail overrides may require a premium license, depending on your proxy settings.
---
@ -170,9 +200,14 @@ Allows requests to proceed without raising a 400 error when content is flagged.
| Parameter | Type | Description |
|---------------------------------------|-----------------|-------------|
| `api_key` | string | Gray Swan Cygnal API key. Reads from `GRAYSWAN_API_KEY` if omitted. |
| `api_base` | string | Override for the Gray Swan API base URL. Defaults to `https://api.grayswan.ai` or `GRAYSWAN_API_BASE`. |
| `mode` | string or list | Guardrail stages (`pre_call`, `during_call`, `post_call`). |
| `optional_params.on_flagged_action` | string | `monitor` (log only), `block` (raise `HTTPException`), or `passthrough` (replace response content with violation message, no 400 error). |
| `.optional_params.violation_threshold`| number (0-1) | Scores at or above this value are considered violations. |
| `optional_params.violation_threshold` | number (0-1) | Scores at or above this value are considered violations. |
| `optional_params.reasoning_mode` | string | `off`, `hybrid`, or `thinking`. Enables Cygnal's reasoning capabilities. |
| `optional_params.categories` | object | Map of custom category names to descriptions. |
| `optional_params.policy_id` | string | Gray Swan policy identifier. |
| `guardrail_timeout` | number | Timeout in seconds for the Cygnal request. Defaults to 30. |
| `fail_open` | boolean | If true, errors contacting Cygnal are logged and the request proceeds; if false, errors propagate. Defaults to treu. |
| `streaming_end_of_stream_only` | boolean | For streaming `post_call`, only send the final assembled response to Cygnal. Defaults to false. |
| `default_on` | boolean | Run the guardrail on every request by default. |

3
docs/my-website/docs/proxy/guardrails/onyx_security.md
View file

@ -128,6 +128,7 @@ guardrails:
mode: ["pre_call", "post_call", "during_call"] # Run at multiple stages
api_key: os.environ/ONYX_API_KEY
api_base: os.environ/ONYX_API_BASE
timeout: 10.0 # Optional, defaults to 10 seconds
```
### Required Parameters
@ -137,6 +138,7 @@ guardrails:
### Optional Parameters
- **`api_base`**: Onyx API base URL (defaults to `https://ai-guard.onyx.security`)
- **`timeout`**: Request timeout in seconds (defaults to `10.0`)
## Environment Variables
@ -145,4 +147,5 @@ You can set these environment variables instead of hardcoding values in your con
```shell
export ONYX_API_KEY="your-api-key-here"
export ONYX_API_BASE="https://ai-guard.onyx.security" # Optional
export ONYX_TIMEOUT=10 # Optional, timeout in seconds
```

6
docs/my-website/docs/proxy/guardrails/quick_start.md
View file

@ -405,14 +405,10 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
## **Proxy Admin Controls**
### ✨ Monitoring Guardrails
### Monitoring Guardrails
Monitor which guardrails were executed and whether they passed or failed. e.g. guardrail going rogue and failing requests we don't intend to fail
:::info
✨ This is an Enterprise only feature [Get a free trial](https://www.litellm.ai/enterprise#trial)
:::
#### Setup

61
docs/my-website/docs/proxy/load_balancing.md
View file

@ -69,6 +69,67 @@ router_settings:
redis_port: 1992
```
## Enforce Model Rate Limits
Strictly enforce RPM/TPM limits set on deployments. When limits are exceeded, requests are blocked **before** reaching the LLM provider with a `429 Too Many Requests` error.
:::info
By default, `rpm` and `tpm` values are only used for **routing decisions** (picking deployments with capacity). With `enforce_model_rate_limits`, they become **hard limits**.
:::
### Quick Start
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
rpm: 60 # 60 requests per minute
tpm: 90000 # 90k tokens per minute
router_settings:
optional_pre_call_checks:
- enforce_model_rate_limits # 👈 Enables strict enforcement
```
### How It Works
| Limit Type | Enforcement | Accuracy |
|------------|-------------|----------|
| **RPM** | Hard limit - blocked at exact threshold | 100% accurate |
| **TPM** | Best-effort - may slightly exceed | Blocked when already over limit |
**Why TPM is best-effort:** Token count is unknown until the LLM responds. TPM is checked before each request (blocks if already over), and tracked after (adds actual tokens used).
### Error Response
```json
{
"error": {
"message": "Model rate limit exceeded. RPM limit=60, current usage=60",
"type": "rate_limit_error",
"code": 429
}
}
```
Response includes `retry-after: 60` header.
### Multi-Instance Deployment
For multiple LiteLLM proxy instances, add Redis to share rate limit state:
```yaml
router_settings:
optional_pre_call_checks:
- enforce_model_rate_limits
redis_host: redis.example.com
redis_port: 6379
redis_password: your-password
```
:::info
Detailed information about [routing strategies can be found here](../routing)
:::

15
docs/my-website/docs/proxy/prometheus.md
View file

@ -121,8 +121,8 @@ Use this to track overall LiteLLM Proxy usage.
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "exception_status", "exception_class", "route"` |
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route"` |
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "user_email", "exception_status", "exception_class", "route", "model_id"` |
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"` |
### Callback Logging Metrics
@ -130,7 +130,12 @@ Monitor failures while shipping logs to downstream callbacks like `s3_v3` cold s
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_callback_logging_failures_metric` | Total number of failed attempts to emit logs to a configured callback. Labels: `"callback_name"`. Use this to alert on callback delivery issues such as repeated failures when writing to `s3_v3`. |
| `litellm_callback_logging_failures_metric` | Total number of failed attempts to emit logs to a configured callback. Labels: `"callback_name"`. Use this to alert on callback delivery issues such as repeated failures when writing to `s3_v3`, `langfuse`, or `langfuse_otel` and other otel providers |
**Supported Callbacks:**
- `S3Logger` - S3 v2 cold storage failures
- `langfuse` - Langfuse logging failures
- `otel` - OpenTelemetry logging failures
## LLM Provider Metrics
@ -191,10 +196,10 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model" |
| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model", "model_id" |
| `litellm_overhead_latency_metric` | Latency overhead (seconds) added by LiteLLM processing - tracked for labels "model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias" |
| `litellm_llm_api_latency_metric` | Latency (seconds) for just the LLM API call - tracked for labels "model", "hashed_api_key", "api_key_alias", "team", "team_alias", "requested_model", "end_user", "user" |
| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias` [Note: only emitted for streaming requests] |
| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias`, `requested_model`, `end_user`, `user`, `model_id` [Note: only emitted for streaming requests] |
## Tracking `end_user` on Prometheus

58
docs/my-website/docs/proxy/request_tags.md Normal file
View file

@ -0,0 +1,58 @@
# Request Tags for Spend Tracking
Add tags to model deployments to track spend by environment, AWS account, or any custom label.
Tags appear in the `request_tags` field of LiteLLM spend logs.
## Config Setup
Set tags on model deployments in `config.yaml`:
```yaml title="config.yaml"
model_list:
- model_name: gpt-4
litellm_params:
model: azure/gpt-4-prod
api_key: os.environ/AZURE_PROD_API_KEY
api_base: https://prod.openai.azure.com/
tags: ["AWS_IAM_PROD"] # 👈 Tag for production
- model_name: gpt-4-dev
litellm_params:
model: azure/gpt-4-dev
api_key: os.environ/AZURE_DEV_API_KEY
api_base: https://dev.openai.azure.com/
tags: ["AWS_IAM_DEV"] # 👈 Tag for development
```
## Make Request
Requests just specify the model - tags are automatically applied:
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
## Spend Logs
The tag from the model config appears in `LiteLLM_SpendLogs`:
```json
{
"request_id": "chatcmpl-abc123",
"request_tags": ["AWS_IAM_PROD"],
"spend": 0.002,
"model": "gpt-4"
}
```
## Related
- [Spend Tracking Overview](cost_tracking.md)
- [Tag Budgets](tag_budgets.md) - Set budget limits per tag

121
docs/my-website/docs/proxy/ui/page_visibility.md Normal file
View file

@ -0,0 +1,121 @@
import Image from '@theme/IdealImage';
# Control Page Visibility for Internal Users
Configure which navigation tabs and pages are visible to internal users (non-admin developers) in the LiteLLM UI.
Use this feature to simplify the UI and control which pages your internal users/developers can see when signing in.
## Overview
By default, all pages accessible to internal users are visible in the navigation sidebar. The page visibility control allows admins to restrict which pages internal users can see, creating a more focused and streamlined experience.
## Configure Page Visibility
### 1. Navigate to Settings
Click the **Settings** icon in the sidebar.
![Navigate to Settings](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/cbb6f272-ab18-4996-b57d-7ed4aad721ea/ascreenshot_ab80f3175b1a41b0bdabdd2cd3980573_text_export.jpeg)
### 2. Go to Admin Settings
Click **Admin Settings** from the settings menu.
![Go to Admin Settings](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/e2b327bf-1cfd-4519-a9ce-8a6ecb2de53a/ascreenshot_23bb1577b3f84d22be78e0faa58dee3d_text_export.jpeg)
### 3. Select UI Settings
Click **UI Settings** to access the page visibility controls.
![Select UI Settings](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/fff0366a-4944-457a-8f6a-e22018dde108/ascreenshot_0e268e8651654e75bb9fb40d2ed366a9_text_export.jpeg)
### 4. Open Page Visibility Configuration
Click **Configure Page Visibility** to expand the configuration panel.
![Open Configuration](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/3a4761d6-145a-4afd-8abf-d92744b9ac9f/ascreenshot_23c16eb79c32481887b879d961f1f00a_text_export.jpeg)
### 5. Select Pages to Make Visible
Check the boxes for the pages you want internal users to see. Pages are organized by category for easy navigation.
![Select Pages](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/b9c96b54-6c20-484f-8b0b-3a86decb5717/ascreenshot_3347ade01ebe4ea390bc7b57e53db43f_text_export.jpeg)
**Available pages include:**
- Virtual Keys
- Playground
- Models + Endpoints
- Agents
- MCP Servers
- Search Tools
- Vector Stores
- Logs
- Teams
- Organizations
- Usage
- Budgets
- And more...
### 6. Save Your Configuration
Click **Save Page Visibility Settings** to apply the changes.
![Save Settings](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/8a215378-44f5-4bb8-b984-06fa2aa03903/ascreenshot_44e7aeebe25a477ba92f73a3ed3df644_text_export.jpeg)
### 7. Verify Changes
Internal users will now only see the selected pages in their navigation sidebar.
![Verify Changes](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/493a7718-b276-40b9-970f-5814054932d9/ascreenshot_ad23b8691f824095ba60256f91ad24f8_text_export.jpeg)
## Reset to Default
To restore all pages to internal users:
1. Open the Page Visibility configuration
2. Click **Reset to Default (All Pages)**
3. Click **Save Page Visibility Settings**
This will clear the restriction and show all accessible pages to internal users.
## API Configuration
You can also configure page visibility programmatically using the API:
### Get Current Settings
```bash
curl -X GET 'http://localhost:4000/ui_settings/get' \
-H 'Authorization: Bearer <your-admin-key>'
```
### Update Page Visibility
```bash
curl -X PATCH 'http://localhost:4000/ui_settings/update' \
-H 'Authorization: Bearer <your-admin-key>' \
-H 'Content-Type: application/json' \
-d '{
"enabled_ui_pages_internal_users": [
"api-keys",
"agents",
"mcp-servers",
"logs",
"teams"
]
}'
```
### Clear Page Visibility Restrictions
```bash
curl -X PATCH 'http://localhost:4000/ui_settings/update' \
-H 'Authorization: Bearer <your-admin-key>' \
-H 'Content-Type: application/json' \
-d '{
"enabled_ui_pages_internal_users": null
}'
```

44
docs/my-website/docs/proxy/ui_logs.md
View file

@ -25,7 +25,10 @@ View Spend, Token Usage, Key, Team Name for Each Request to LiteLLM
## Tracking - Request / Response Content in Logs Page
If you want to view request and response content on LiteLLM Logs, you need to opt in with this setting
If you want to view request and response content on LiteLLM Logs, you can enable it in either place:
- **From the UI (no restart):** Use [UI Spend Log Settings](./ui_spend_log_settings.md) — open Logs → Settings → enable "Store Prompts in Spend Logs" → Save. Takes effect immediately and overrides config.
- **From config:** Add this to your `proxy_config.yaml` (requires restart):
```yaml
general_settings:
@ -34,6 +37,40 @@ general_settings:
<Image img={require('../../img/ui_request_logs_content.png')}/>
## Tracing Tools
View which tools were provided and called in your completion requests.
<Image img={require('../../img/ui_tools.png')}/>
**Example:** Make a completion request with tools:
```bash
curl -X POST 'http://localhost:4000/chat/completions' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "What is the weather?"}],
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
}
}
}
}
]
}'
```
Check the Logs page to see all tools provided and which ones were called.
## Stop storing Error Logs in DB
@ -57,7 +94,10 @@ general_settings:
If you're storing spend logs, it might be a good idea to delete them regularly to keep the database fast.
LiteLLM lets you configure this in your `proxy_config.yaml`:
You can set the retention period in either place:
- **From the UI (no restart):** [UI Spend Log Settings](./ui_spend_log_settings.md) — Logs → Settings → set Retention Period → Save.
- **From config:** Add the following to your `proxy_config.yaml` (requires restart):
```yaml
general_settings:

92
docs/my-website/docs/proxy/ui_spend_log_settings.md Normal file
View file

@ -0,0 +1,92 @@
import Image from '@theme/IdealImage';
# UI Spend Log Settings
Configure spend log behavior directly from the Admin UI—no config file edits or proxy restart required. This is especially useful for cloud deployments where updating the config is difficult or requires a long release process.
## Overview
Previously, spend log options (such as storing request/response content and retention period) had to be set in `proxy_config.yaml` under `general_settings`. Changing them required editing the config and restarting the proxy, which was a pain point for users-especially in cloud environments—who don't have easy access to the config or whose deployment process makes config updates slow.
<Image img={require('../../img/ui_spend_logs_settings.png')} />
**UI Spend Log Settings** lets you:
- **Store prompts in spend logs** – Enable or disable storing request and response content in the spend logs table (only affects logs created after you change the setting)
- **Set retention period** – Configure how long spend logs are kept before automatic cleanup (e.g. `7d`, `30d`)
- **Apply changes immediately** – No proxy restart needed; settings take effect for new requests as soon as you save
:::warning UI overrides config
Settings changed in the UI **override** the values in your config file. For example, if `store_prompts_in_spend_logs` is explicitly set to `false` in `general_settings`, turning it on in the UI will still enable storing prompts. Use the UI when you want runtime control without redeploying.
:::
## Settings You Can Configure
| Setting | Description |
| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| **Store Prompts in Spend Logs** | When enabled, request messages and response content are stored for **new** spend logs so you can view them in the Logs UI. Logs created before you enabled this will not have request/response content. When disabled, only metadata (e.g. tokens, cost, model) is stored for new logs. |
| **Retention Period** | Maximum time to keep spend logs before they are automatically deleted (e.g. `7d`, `30d`). Optional; if not set, logs are retained according to your config or default behavior. |
The same options can be set in config via [general_settings](./config_settings.md#general_settings---reference) (`store_prompts_in_spend_logs`, `maximum_spend_logs_retention_period`). Values set in the UI take precedence.
## How to Configure Spend Log Settings in the UI
### 1. Open the Logs page
Navigate to the Admin UI (e.g. `http://localhost:4000/ui` or your `PROXY_BASE_URL/ui`) and click **Logs**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-31/815f4ab2-4b8c-4dfe-be39-689fd6e12167/ascreenshot_eaaeba1507b441408e0df8bf94bc70cc_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-31/815f4ab2-4b8c-4dfe-be39-689fd6e12167/ascreenshot_666628f5e62443688a58b7cee7d7559b_text_export.jpeg)
### 2. Open Logs settings
Click the **Settings** (gear) icon on the Logs page to open the spend log settings panel.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-31/303077bd-80a0-4f3b-9dc1-4abb90af117f/ascreenshot_63f5dc21a545489ea9266f3bd3dc8455_text_export.jpeg)
### 3. Enable Store Prompts in Spend Logs (optional)
Turn on **Store Prompts in Spend Logs** if you want request and response content to be stored for new requests and visible when you open those log entries. This only affects logs created after you enable it; existing logs will not gain request/response content. Leave it off if you only need metadata (tokens, cost, model, etc.).
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-31/a25d0051-4b34-4270-99d6-6e8ae0d2936a/ascreenshot_374605862aad42c89a98da7bad910f58_text_export.jpeg)
### 4. Set the retention period (optional)
Optionally set the **Retention Period** (e.g. `7d`, `30d`) to control how long spend logs are kept before automatic cleanup. Uses the same format as the config option `maximum_spend_logs_retention_period`.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-31/87086197-b082-4339-b798-37410f47d9ac/ascreenshot_564da14f492540ae8b0b782cfedceff9_text_export.jpeg)
### 5. Save settings
Click **Save Settings**. Changes take effect immediately for new requests; no proxy restart is required. Existing logs are not updated.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-31/8cfd82c1-0ff4-4561-a806-33a7998cf0fd/ascreenshot_673f6155b17f45ee9b80fabdfc42a4ee_text_export.jpeg)
### 6. Verify: view request and response in a log
After enabling **Store Prompts in Spend Logs**, make a new request through the proxy, then open that log entry (or any other log created after you enabled the setting). The log details view will include the request and response content. Logs that existed before you turned the setting on will not have this content.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-31/0fbec553-9a11-4f4f-8a1d-f969bb316c70/ascreenshot_62ecbcea97ea4a4abaa460d76e2cf924_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-31/30e7ea4d-2c03-4b96-88a9-eeee565eaf16/ascreenshot_c00ad6aa75b54b4988a1450647a76f6b_text_export.jpeg)
## Use Cases
### Cloud and managed deployments
When the proxy runs in a managed or cloud environment, config may be in a separate repo, require a long release, or be controlled by another team. Using the UI lets you change spend log behavior (e.g. enable prompt storage for debugging or set retention) without going through that process.
### Quick toggles for debugging
Temporarily enable **Store Prompts in Spend Logs** to inspect request/response content on new requests when debugging, then turn it off again from the UI without editing config or restarting. Only logs created while the setting was on will contain the content.
### Retention without redeploying
Adjust how long spend logs are retained (e.g. shorten to reduce storage or extend for compliance) and have the new retention period and cleanup job take effect immediately.
## Related Documentation
- [Getting Started with UI Logs](./ui_logs.md) – Overview of what gets logged and config-based options
- [Config Settings](./config_settings.md) – `store_prompts_in_spend_logs`, `disable_spend_logs`, `maximum_spend_logs_retention_period` in `general_settings`
- [Spend Logs Deletion](./spend_logs_deletion.md) – How retention and cleanup work

78
docs/my-website/docs/rag_ingest.md
View file

@ -5,7 +5,7 @@ All-in-one document ingestion pipeline: **Upload → Chunk → Embed → Vector
| Feature | Supported |
|---------|-----------|
| Logging | Yes |
| Supported Providers | `openai`, `bedrock`, `vertex_ai`, `gemini` |
| Supported Providers | `openai`, `bedrock`, `vertex_ai`, `gemini`, `s3_vectors` |
:::tip
After ingesting documents, use [/rag/query](./rag_query.md) to search and generate responses with your ingested content.
@ -75,6 +75,31 @@ curl -X POST "http://localhost:4000/v1/rag/ingest" \
}"
```
### AWS S3 Vectors
```bash showLineNumbers title="Ingest to S3 Vectors"
curl -X POST "http://localhost:4000/v1/rag/ingest" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d "{
\"file\": {
\"filename\": \"document.txt\",
\"content\": \"$(base64 -i document.txt)\",
\"content_type\": \"text/plain\"
},
\"ingest_options\": {
\"embedding\": {
\"model\": \"text-embedding-3-small\"
},
\"vector_store\": {
\"custom_llm_provider\": \"s3_vectors\",
\"vector_bucket_name\": \"my-embeddings\",
\"aws_region_name\": \"us-west-2\"
}
}
}"
```
## Response
```json
@ -265,6 +290,57 @@ When `vector_store_id` is omitted, LiteLLM automatically creates:
4. Install: `pip install 'google-cloud-aiplatform>=1.60.0'`
:::
### vector_store (AWS S3 Vectors)
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `custom_llm_provider` | string | - | `"s3_vectors"` |
| `vector_bucket_name` | string | **required** | S3 vector bucket name |
| `index_name` | string | auto-create | Vector index name |
| `dimension` | integer | auto-detect | Vector dimension (auto-detected from embedding model) |
| `distance_metric` | string | `cosine` | Distance metric: `cosine` or `euclidean` |
| `non_filterable_metadata_keys` | array | `["source_text"]` | Metadata keys excluded from filtering |
| `aws_region_name` | string | `us-west-2` | AWS region |
| `aws_access_key_id` | string | env | AWS access key |
| `aws_secret_access_key` | string | env | AWS secret key |
:::info S3 Vectors Auto-Creation
When `index_name` is omitted, LiteLLM automatically creates:
- S3 vector bucket (if it doesn't exist)
- Vector index with auto-detected dimensions from your embedding model
**Dimension Auto-Detection**: The vector dimension is automatically detected by making a test embedding request to your specified model. No need to manually specify dimensions!
**Supported Embedding Models**: Works with any LiteLLM-supported embedding model (OpenAI, Cohere, Bedrock, Azure, etc.)
:::
**Example with auto-detection:**
```json
{
"embedding": {
"model": "text-embedding-3-small" // Dimension auto-detected as 1536
},
"vector_store": {
"custom_llm_provider": "s3_vectors",
"vector_bucket_name": "my-embeddings"
}
}
```
**Example with custom embedding provider:**
```json
{
"embedding": {
"model": "cohere/embed-english-v3.0" // Dimension auto-detected as 1024
},
"vector_store": {
"custom_llm_provider": "s3_vectors",
"vector_bucket_name": "my-embeddings",
"distance_metric": "cosine"
}
}
```
## Input Examples
### File (Base64)

19
docs/my-website/docs/realtime.md
View file

@ -3,13 +3,15 @@ import TabItem from '@theme/TabItem';
# /realtime
Use this to loadbalance across Azure + OpenAI.
Use this to loadbalance across Azure + OpenAI + xAI and more.
Supported Providers:
- OpenAI
- Azure
- xAI ([see full docs](/docs/providers/xai_realtime))
- Google AI Studio (Gemini)
- Vertex AI
- Bedrock
## Proxy Usage
@ -45,6 +47,21 @@ model_list:
api_key: os.environ/OPENAI_API_KEY
```
</TabItem>
<TabItem value="xai" label="xAI Grok Voice Agent">
```yaml
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-4-1-fast-non-reasoning
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
```
**[See full xAI Realtime documentation →](/docs/providers/xai_realtime)**
</TabItem>
</Tabs>

41
docs/my-website/docs/routing.md
View file

@ -830,6 +830,12 @@ asyncio.run(router_acompletion())
</TabItem>
</Tabs>
## Traffic Mirroring / Silent Experiments
Traffic mirroring allows you to "mimic" production traffic to a secondary (silent) model for evaluation purposes. The silent model's response is gathered in the background and does not affect the latency or result of the primary request.
[**See detailed guide on A/B Testing - Traffic Mirroring here**](./traffic_mirroring.md)
## Basic Reliability
### Deployment Ordering (Priority)
@ -1582,11 +1588,13 @@ Get a slack webhook url from https://api.slack.com/messaging/webhooks
Initialize an `AlertingConfig` and pass it to `litellm.Router`. The following code will trigger an alert because `api_key=bad-key` which is invalid
```python
from litellm.router import AlertingConfig
import litellm
from litellm.router import Router
from litellm.types.router import AlertingConfig
import os
import asyncio
router = litellm.Router(
router = Router(
model_list=[
{
"model_name": "gpt-3.5-turbo",
@ -1597,17 +1605,28 @@ router = litellm.Router(
}
],
alerting_config= AlertingConfig(
alerting_threshold=10, # threshold for slow / hanging llm responses (in seconds). Defaults to 300 seconds
webhook_url= os.getenv("SLACK_WEBHOOK_URL") # webhook you want to send alerts to
alerting_threshold=10,
webhook_url= "https:/..."
),
)
try:
await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
)
except:
pass
async def main():
print(f"\n=== Configuration ===")
print(f"Slack logger exists: {router.slack_alerting_logger is not None}")
try:
await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
)
except Exception as e:
print(f"\n=== Exception caught ===")
print(f"Waiting 10 seconds for alerts to be sent via periodic flush...")
await asyncio.sleep(10)
print(f"\n=== After waiting ===")
print(f"Alert should have been sent to Slack!")
asyncio.run(main())
```
## Track cost for Azure Deployments

83
docs/my-website/docs/traffic_mirroring.md Normal file
View file

@ -0,0 +1,83 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# A/B Testing - Traffic Mirroring
Traffic mirroring allows you to "mimic" production traffic to a secondary (silent) model for evaluation purposes. The silent model's response is gathered in the background and does not affect the latency or result of the primary request.
This is useful for:
- Testing a new model's performance on production prompts before switching.
- Comparing costs and latency between different providers.
- Debugging issues by mirroring traffic to a more verbose model.
## Quick Start
To enable traffic mirroring, add `silent_model` to the `litellm_params` of a deployment.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import Router
model_list = [
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/chatgpt-v-2",
"api_key": "...",
"silent_model": "gpt-4" # 👈 Mirror traffic to gpt-4
},
},
{
"model_name": "gpt-4",
"litellm_params": {
"model": "openai/gpt-4",
"api_key": "..."
},
}
]
router = Router(model_list=model_list)
# The request to "gpt-3.5-turbo" will trigger a background call to "gpt-4"
response = await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "How does traffic mirroring work?"}]
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
Add `silent_model` to your `config.yaml`:
```yaml
model_list:
- model_name: primary-model
litellm_params:
model: azure/gpt-35-turbo
api_key: os.environ/AZURE_API_KEY
silent_model: evaluation-model # 👈 Mirror traffic here
- model_name: evaluation-model
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
```
</TabItem>
</Tabs>
## How it works
1. **Request Received**: A request is made to a model group (e.g. `primary-model`).
2. **Deployment Picked**: LiteLLM picks a deployment from the group.
3. **Primary Call**: LiteLLM makes the call to the primary deployment.
4. **Mirroring**: If `silent_model` is present, LiteLLM triggers a background call to that model.
- For **Sync** calls: Uses a shared thread pool.
- For **Async** calls: Uses `asyncio.create_task`.
5. **Isolation**: The background call uses a `deepcopy` of the original request parameters and sets `metadata["is_silent_experiment"] = True`. It also strips out logging IDs to prevent collisions in usage tracking.
## Key Features
- **Latency Isolation**: The primary request returns as soon as it's ready. The background (silent) call does not block.
- **Unified Logging**: Background calls are processed via the Router, meaning they are automatically logged to your configured observability tools (Langfuse, S3, etc.).
- **Evaluation**: Use the `is_silent_experiment: True` flag in your logs to filter and compare results between the primary and mirrored calls.

113
docs/my-website/docs/troubleshoot/prisma_migrations.md Normal file
View file

@ -0,0 +1,113 @@
# Troubleshooting Prisma Migration Errors
Common Prisma migration issues encountered when upgrading or downgrading LiteLLM proxy versions, and how to fix them.
## How Prisma Migrations Work in LiteLLM
- LiteLLM uses [Prisma](https://www.prisma.io/) to manage its PostgreSQL database schema.
- Migration history is tracked in the `_prisma_migrations` table in your database.
- When LiteLLM starts, it runs `prisma migrate deploy` to apply any new migrations.
- Upgrading LiteLLM applies all migrations added since your last applied version.
## Common Errors
### 1. `relation "X" does not exist`
**Example error:**
```
ERROR: relation "LiteLLM_DeletedTeamTable" does not exist
Migration: 20260116142756_update_deleted_keys_teams_table_routing_settings
```
**Cause:** This typically happens after a version rollback. The `_prisma_migrations` table still records migrations from the newer version as "applied," but the underlying database tables were modified, dropped, or never fully created.
**How to fix:**
#### Step 1 — Delete the failed migration entry and restart
Remove the problematic migration from the history so it can be re-applied:
```sql
-- View recent migrations
SELECT migration_name, finished_at, rolled_back_at, logs
FROM "_prisma_migrations"
ORDER BY started_at DESC
LIMIT 10;
-- Delete the failed migration entry
DELETE FROM "_prisma_migrations"
WHERE migration_name = '<failed_migration_name>';
```
After deleting the entry, restart LiteLLM — it will re-apply the migration on startup.
#### Step 2 — If that doesn't work, use `prisma db push`
If deleting the migration entry and restarting doesn't resolve the issue, sync the schema directly:
```bash
DATABASE_URL="<your_database_url>" prisma db push
```
This bypasses migration history and forces the database schema to match the Prisma schema.
---
### 2. `New migrations cannot be applied before the error is recovered from`
**Cause:** A previous migration failed (recorded with an error in `_prisma_migrations`), and Prisma refuses to apply any new migrations until the failure is resolved.
**How to fix:**
1. Find the failed migration:
```sql
SELECT migration_name, finished_at, rolled_back_at, logs
FROM "_prisma_migrations"
WHERE finished_at IS NULL OR rolled_back_at IS NOT NULL
ORDER BY started_at DESC;
```
2. Delete the failed entry and restart LiteLLM:
```sql
DELETE FROM "_prisma_migrations"
WHERE migration_name = '<failed_migration_name>';
```
3. If that doesn't work, use `prisma db push`:
```bash
DATABASE_URL="<your_database_url>" prisma db push
```
---
### 3. Migration state mismatch after version rollback
**Cause:** You upgraded to version X (new migrations applied), rolled back to version Y, then upgraded again. The `_prisma_migrations` table has stale entries for migrations that were partially applied or correspond to a schema state that no longer exists.
**Fix:**
1. Inspect the migration table for problematic entries:
```sql
SELECT migration_name, started_at, finished_at, rolled_back_at, logs
FROM "_prisma_migrations"
ORDER BY started_at DESC
LIMIT 20;
```
2. For each migration that shouldn't be there (i.e., from the version you rolled back from), delete the entry:
```sql
DELETE FROM "_prisma_migrations" WHERE migration_name = '<migration_name>';
```
3. Restart LiteLLM to re-run migrations.
4. If that doesn't work, use `prisma db push`:
```bash
DATABASE_URL="<your_database_url>" prisma db push
```

115
docs/my-website/docs/tutorials/claude_agent_sdk.md Normal file
View file

@ -0,0 +1,115 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Claude Agent SDK with LiteLLM
Use Anthropic's Claude Agent SDK with any LLM provider through LiteLLM Proxy.
The Claude Agent SDK provides a high-level interface for building AI agents. By pointing it to LiteLLM, you can use the same agent code with OpenAI, Bedrock, Azure, Vertex AI, or any other provider.
## Quick Start
### 1. Install Dependencies
```bash
pip install claude-agent-sdk
```
### 2. Start LiteLLM Proxy
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: bedrock-claude-sonnet-3.5
litellm_params:
model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-opus-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-nova-premier
litellm_params:
model: "bedrock/amazon.nova-premier-v1:0"
aws_region_name: "us-east-1"
```
```bash
litellm --config config.yaml
```
### 3. Point Agent SDK to LiteLLM
| Environment Variable | Value | Description |
|---------------------|-------|-------------|
| `ANTHROPIC_BASE_URL` | `http://localhost:4000` | LiteLLM proxy URL |
| `ANTHROPIC_API_KEY` | `sk-1234` | Your LiteLLM API key (not Anthropic key) |
```python title="agent.py" showLineNumbers
import os
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
# Point to LiteLLM proxy (not Anthropic)
os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key
# Configure agent with any model from your config
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant.",
model="bedrock-claude-sonnet-4", # Use any model from config.yaml
max_turns=20,
)
async with ClaudeSDKClient(options=options) as client:
await client.query("What is LiteLLM?")
async for msg in client.receive_response():
if hasattr(msg, 'content'):
for content_block in msg.content:
if hasattr(content_block, 'text'):
print(content_block.text, end='', flush=True)
```
## Why Use LiteLLM with Agent SDK?
| Feature | Benefit |
|---------|---------|
| **Multi-Provider** | Use the same agent code with OpenAI, Bedrock, Azure, Vertex AI, etc. |
| **Cost Tracking** | Track spending across all agent conversations |
| **Rate Limiting** | Set budgets and limits on agent usage |
| **Load Balancing** | Distribute requests across multiple API keys or regions |
| **Fallbacks** | Automatically retry with different models if one fails |
## Complete Example
See our [cookbook example](https://github.com/BerriAI/litellm/tree/main/cookbook/anthropic_agent_sdk) for a complete interactive CLI agent that:
- Streams responses in real-time
- Switches between models dynamically
- Fetches available models from the proxy
```bash
# Clone and run the example
git clone https://github.com/BerriAI/litellm.git
cd litellm/cookbook/anthropic_agent_sdk
pip install -r requirements.txt
python main.py
```
## Related Resources
- [Claude Agent SDK Documentation](https://github.com/anthropics/anthropic-agent-sdk)
- [LiteLLM Proxy Quick Start](../proxy/quick_start)
- [Complete Cookbook Example](https://github.com/BerriAI/litellm/tree/main/cookbook/anthropic_agent_sdk)

4
docs/my-website/docs/tutorials/claude_code_plugin_marketplace.md
View file

@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Claude Code Plugin Marketplace
# Claude Code Plugin Marketplace (Managed Skills)
LiteLLM AI Gateway acts as a central registry for Claude Code plugins. Admins can govern which plugins are available across the organization, and engineers can discover and install approved plugins from a single source.
@ -252,7 +252,7 @@ curl -X POST http://localhost:4000/claude-code/plugins \
}'
```
### 3. Share with Your Team
### 3. Use in Claude Code
Send engineers the marketplace URL:

99
docs/my-website/docs/tutorials/copilotkit_sdk.md Normal file
View file

@ -0,0 +1,99 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# CopilotKit SDK with LiteLLM
Use CopilotKit SDK with any LLM provider through LiteLLM Proxy.
> **Note:** CopilotKit SDK integration with LiteLLM Proxy works with LiteLLM v1.81.7-nightly or higher.
## Quick Start
### 1. Add Model to Config
```yaml title="config.yaml"
model_list:
- model_name: claude-sonnet-4-5
litellm_params:
model: "anthropic/claude-sonnet-4-5-20250514-v1:0"
api_key: "os.environ/ANTHROPIC_API_KEY"
```
### 2. Start LiteLLM Proxy
```bash
litellm --config config.yaml
```
### 3. Use CopilotKit SDK
```typescript
import OpenAI from "openai";
import {
CopilotRuntime,
OpenAIAdapter,
copilotRuntimeNextJSAppRouterEndpoint,
} from "@copilotkit/runtime";
import { NextRequest } from "next/server";
const model = "claude-sonnet-4-5";
const openai = new OpenAI({
apiKey: process.env.OPENAI_API_KEY || "sk-12345",
baseURL: process.env.OPENAI_BASE_URL || "http://localhost:4000/v1",
});
const serviceAdapter = new OpenAIAdapter({ openai, model });
const runtime = new CopilotRuntime();
export const POST = async (req: NextRequest) => {
const { handleRequest } = copilotRuntimeNextJSAppRouterEndpoint({
runtime,
serviceAdapter,
endpoint: "/api/copilotkit",
});
return handleRequest(req);
};
```
### 4. Test
```bash
curl -X POST http://localhost:3000/api/copilotkit \
-H "Content-Type: application/json" \
-d '{
"method": "agent/run",
"params": {
"agentId": "default"
},
"runId": "your_run_id",
"threadId": "your_thread_id",
"runId": ""your_run_id"",
"tools": [],
"context": [],
"forwardedProps": {},
"state": {},
"messages": [
{
"id": "166e573e-f7c6-4c0f-8685-04dbefec18be",
"content": "Hi",
"role": "user"
}
]
}
}'
```
## Environment Variables
| Variable | Value | Description |
|----------|-------|-------------|
| `OPENAI_API_KEY` | `sk-12345` | Your LiteLLM API key |
| `OPENAI_BASE_URL` | `http://localhost:4000/v1` | LiteLLM proxy URL |
## Related Resources
- [CopilotKit Documentation](https://docs.copilotkit.ai)
- [LiteLLM Proxy Quick Start](../proxy/quick_start)

190
docs/my-website/docs/tutorials/livekit_xai_realtime.md Normal file
View file

@ -0,0 +1,190 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# LiveKit xAI Realtime Voice Agent
Use LiveKit's xAI Grok Voice Agent plugin with LiteLLM Proxy to build low-latency voice AI agents.
The LiveKit Agents framework provides tools for building real-time voice and video AI applications. By routing through LiteLLM Proxy, you get unified access to multiple realtime voice providers, cost tracking, rate limiting, and more.
## Quick Start
### 1. Install Dependencies
```bash
pip install livekit-agents[xai]
```
### 2. Start LiteLLM Proxy
Create a config file with your xAI realtime model:
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-2-vision-1212
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
litellm_settings:
drop_params: True
general_settings:
master_key: sk-1234 # Change this to a secure key
```
Start the proxy:
```bash
litellm --config config.yaml --port 4000
```
### 3. Configure LiveKit xAI Plugin
Point LiveKit's xAI plugin to your LiteLLM proxy:
```python
from livekit.plugins import xai
# Configure xAI to use LiteLLM proxy
model = xai.realtime.RealtimeModel(
voice="ara", # Voice option
api_key="sk-1234", # Your LiteLLM proxy master key
base_url="http://localhost:4000", # LiteLLM proxy URL
)
```
## Complete Example
Here's a complete working example:
<Tabs>
<TabItem value="python" label="Python Client">
```python
#!/usr/bin/env python3
"""
Simple xAI realtime voice agent through LiteLLM proxy.
"""
import asyncio
import json
import websockets
PROXY_URL = "ws://localhost:4000/v1/realtime"
API_KEY = "sk-1234"
MODEL = "grok-voice-agent"
async def run_voice_agent():
"""Connect to xAI realtime API through LiteLLM proxy"""
url = f"{PROXY_URL}?model={MODEL}"
headers = {"Authorization": f"Bearer {API_KEY}"}
async with websockets.connect(url, extra_headers=headers) as ws:
# Wait for initial connection event
initial = json.loads(await ws.recv())
print(f"✅ Connected: {initial['type']}")
# Send user message
await ws.send(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "Hello! Tell me a joke."
}]
}
}))
# Request response
await ws.send(json.dumps({
"type": "response.create",
"response": {"modalities": ["text", "audio"]}
}))
# Collect response
transcript = []
async for message in ws:
event = json.loads(message)
# Capture text response
if event['type'] == 'response.output_audio_transcript.delta':
transcript.append(event['delta'])
print(event['delta'], end='', flush=True)
# Done when response completes
elif event['type'] == 'response.done':
break
print(f"\n\n✅ Full response: {''.join(transcript)}")
if __name__ == "__main__":
asyncio.run(run_voice_agent())
```
</TabItem>
<TabItem value="livekit" label="LiveKit Agent">
```python
from livekit.agents import Agent, AgentSession, WorkerOptions, cli
from livekit.plugins import xai
class VoiceAgent(Agent):
def __init__(self):
super().__init__(
instructions="You are a helpful voice assistant.",
llm=xai.realtime.RealtimeModel(
voice="ara",
api_key="sk-1234",
base_url="http://localhost:4000",
),
)
if __name__ == "__main__":
cli.run_app(
WorkerOptions(
agent_factory=VoiceAgent,
)
)
```
</TabItem>
</Tabs>
## Running the Example
1. **Start LiteLLM Proxy** (if not already running):
```bash
litellm --config config.yaml --port 4000
```
2. **Run the example**:
```bash
python your_script.py
```
## Expected Output
```
✅ Connected: conversation.created
Hello! Here's a joke for you: Why don't scientists trust atoms?
Because they make up everything!
✅ Full response: Hello! Here's a joke for you: Why don't scientists trust atoms? Because they make up everything!
```
## Complete Working Example
**[LiveKit Agent SDK Cookbook](https://github.com/BerriAI/litellm/tree/main/cookbook/livekit_agent_sdk)**
## Learn More
- [xAI Realtime API](/docs/providers/xai_realtime)
- [LiveKit xAI Plugin](https://docs.livekit.io/agents/models/realtime/plugins/xai/)
- [LiteLLM Realtime API](/docs/realtime)

BIN
docs/my-website/img/a2a_agent_spend.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 184 KiB

BIN
docs/my-website/img/a2a_trace_grouping.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 388 KiB

BIN
docs/my-website/img/ui_spend_logs_settings.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 343 KiB

BIN
docs/my-website/img/ui_tools.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 420 KiB

423
docs/my-website/release_notes/v1.81.3-stable/index.md Normal file
View file

@ -0,0 +1,423 @@
---
title: "v1.81.3-stable - Performance - 25% CPU Usage Reduction"
slug: "v1-81-3"
date: 2026-01-26T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.81.3.rc.2
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.81.3.rc.2
```
</TabItem>
</Tabs>
---
## New Models / Updated Models
### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Deprecation Date |
| -------- | ----- | -------------- | ------------------- | -------------------- | ---------------- |
| OpenAI | `gpt-audio`, `gpt-audio-2025-08-28` | 128K | $32/1M audio tokens, $2.5/1M text tokens | $64/1M audio tokens, $10/1M text tokens | - |
| OpenAI | `gpt-audio-mini`, `gpt-audio-mini-2025-08-28` | 128K | $10/1M audio tokens, $0.6/1M text tokens | $20/1M audio tokens, $2.4/1M text tokens | - |
| Deepinfra, Vertex AI, Google AI Studio, OpenRouter, Vercel AI Gateway | `gemini-2.0-flash-001`, `gemini-2.0-flash` | - | - | - | 2026-03-31 |
| Groq | `openai/gpt-oss-120b` | 131K | 0.075/1M cache read | 0.6/1M output tokens | - |
| Groq | `groq/openai/gpt-oss-20b` | 131K | 0.0375/1M cache read, $0.075/1M text tokens | 0.3/1M output tokens | - |
| Vertex AI | `gemini-2.5-computer-use-preview-10-2025` | 128K | $1.25 | $10 | - |
| Azure AI | `claude-haiku-4-5` | $1.25/1M cache read, $2/1M cache read above 1 hr, $0.1/1M text tokens | $5/1M output tokens | - |
| Azure AI | `claude-sonnet-4-5` | $3.75/1M cache read, $6/1M cache read above 1 hr, $3/1M text tokens | $15/1M output tokens | - |
| Azure AI | `claude-opus-4-5` | $6.25/1M cache read, $10/1M cache read above 1 hr, $0.5/1M text tokens | $25/1M output tokens | - |
| Azure AI | `claude-opus-4-1` | $18.75/1M cache read, $30/1M cache read above 1 hr, $1.5/1M text tokens | $75/1M output tokens | - |
### Features
- **[OpenAI](../../docs/providers/openai)**
- Add gpt-audio and gpt-audio-mini models to pricing - [PR #19509](https://github.com/BerriAI/litellm/pull/19509)
- correct audio token costs for gpt-4o-audio-preview models - [PR #19500](https://github.com/BerriAI/litellm/pull/19500)
- Limit stop sequence as per openai spec (ensures JetBrains IDE compatibility) - [PR #19562](https://github.com/BerriAI/litellm/pull/19562)
- **[VertexAI](../../docs/providers/vertex)**
- Docs - Google Workload Identity Federation (WIF) support - [PR #19320](https://github.com/BerriAI/litellm/pull/19320)
- **[Agentcore](../../docs/providers/bedrock_agentcore)**
- Fixes streaming issues with AWS Bedrock AgentCore where responses would stop after the first chunk, particularly affecting OAuth-enabled agents - [PR #17141](https://github.com/BerriAI/litellm/pull/17141)
- **[Chatgpt](../../docs/providers/chatgpt)**
- Adds support for calling chatgpt subscription via LiteLLM - [PR #19030](https://github.com/BerriAI/litellm/pull/19030)
- Adds responses API bridge support for chatgpt subscription provider - [PR #19030](https://github.com/BerriAI/litellm/pull/19030)
- **[Bedrock](../../docs/providers/bedrock)**
- support for output format for bedrock invoke via v1/messages - [PR #19560](https://github.com/BerriAI/litellm/pull/19560)
- **[Azure](../../docs/providers/azure/azure)**
- Add support for Azure OpenAI v1 API - [PR #19313](https://github.com/BerriAI/litellm/pull/19313)
- preserve content_policy_violation details for images (#19328) - [PR #19372](https://github.com/BerriAI/litellm/pull/19372)
- Support OpenAI-format nested tool definitions for Responses API - [PR #19526](https://github.com/BerriAI/litellm/pull/19526)
- **Gemini([Vertex AI](../../docs/providers/vertex), [Google AI Studio](../../docs/providers/gemini))**
- use responseJsonSchema for Gemini 2.0+ models - [PR #19314](https://github.com/BerriAI/litellm/pull/19314)
- **[Volcengine](../../docs/providers/volcano)**
- Support Volcengine responses api - [PR #18508](https://github.com/BerriAI/litellm/pull/18508)
- **[Anthropic](../../docs/providers/anthropic)**
- Add Support for calling Claude Code Max subscriptions via LiteLLM - [PR #19453](https://github.com/BerriAI/litellm/pull/19453)
- Add Structured output for /v1/messages with Anthropic API, Azure Anthropic API, Bedrock Converse - [PR #19545](https://github.com/BerriAI/litellm/pull/19545)
- **[Brave Search](../../docs/search/brave)**
- New Search provider - [PR #19433](https://github.com/BerriAI/litellm/pull/19433)
- **Sarvam ai**
- Add support for new sarvam models - [PR #19479](https://github.com/BerriAI/litellm/pull/19479)
- **[GMI](../../docs/providers/gmi)**
- add GMI Cloud provider support - [PR #19376](https://github.com/BerriAI/litellm/pull/19376)
### Bug Fixes
- **[Anthropic](../../docs/providers/anthropic)**
- Fix anthropic-beta sent client side being overridden instead of appended to - [PR #19343](https://github.com/BerriAI/litellm/pull/19343)
- Filter out unsupported fields from JSON schema for Anthropic's output_format API - [PR #19482](https://github.com/BerriAI/litellm/pull/19482)
- **[Bedrock](../../docs/providers/bedrock)**
- Expose stability models via /image_edits endpoint and ensure proper request transformation - [PR #19323](https://github.com/BerriAI/litellm/pull/19323)
- Claude Code x Bedrock Invoke fails with advanced-tool-use-2025-11-20 - [PR #19373](https://github.com/BerriAI/litellm/pull/19373)
- deduplicate tool calls in assistant history - [PR #19324](https://github.com/BerriAI/litellm/pull/19324)
- fix: correct us.anthropic.claude-opus-4-5 In-region pricing - [PR #19310](https://github.com/BerriAI/litellm/pull/19310)
- Fix request validation errors when using Claude 4 via bedrock invoke - [PR #19381](https://github.com/BerriAI/litellm/pull/19381)
- Handle thinking with tool calls for Claude 4 models - [PR #19506](https://github.com/BerriAI/litellm/pull/19506)
- correct streaming choice index for tool calls - [PR #19506](https://github.com/BerriAI/litellm/pull/19506)
- **[Ollama](../../docs/providers/ollama)**
- Fix tool call errors due with improved message extraction - [PR #19369](https://github.com/BerriAI/litellm/pull/19369)
- **[VertexAI](../../docs/providers/vertex)**
- Removed optional vertex_count_tokens_location param before request is sent to vertex - [PR #19359](https://github.com/BerriAI/litellm/pull/19359)
- **Gemini([Vertex AI](../../docs/providers/vertex), [Google AI Studio](../../docs/providers/gemini))**
- Supports setting media_resolution and fps parameters on each video file, when using Gemini video understanding - [PR #19273](https://github.com/BerriAI/litellm/pull/19273)
- handle reasoning_effort as dict from OpenAI Agents SDK - [PR #19419](https://github.com/BerriAI/litellm/pull/19419)
- add file content support in tool results - [PR #19416](https://github.com/BerriAI/litellm/pull/19416)
- **[Azure](../../docs/providers/azure_ai)**
- Fix Azure AI costs for Anthropic models - [PR #19530](https://github.com/BerriAI/litellm/pull/19530)
- **[Giga Chat](../../docs/providers/gigachat)**
- Add tool choice mapping - [PR #19645](https://github.com/BerriAI/litellm/pull/19645)
---
## AI API Endpoints (LLMs, MCP, Agents)
### Features
- **[Files API](../../docs/files_endpoints)**
- Add managed files support when load_balancing is True - [PR #19338](https://github.com/BerriAI/litellm/pull/19338)
- **[Claude Plugin Marketplace](../../docs/tutorials/claude_code_plugin_marketplace)**
- Add self hosted Claude Code Plugin Marketplace - [PR #19378](https://github.com/BerriAI/litellm/pull/19378)
- **[MCP](../../docs/mcp)**
- Add MCP Protocol version 2025-11-25 support - [PR #19379](https://github.com/BerriAI/litellm/pull/19379)
- Log MCP tool calls and list tools in the LiteLLM Spend Logs table for easier debugging - [PR #19469](https://github.com/BerriAI/litellm/pull/19469)
- **[Vertex AI](../../docs/providers/vertex)**
- Ensure only anthropic betas are forwarded down to LLM API (by default) - [PR #19542](https://github.com/BerriAI/litellm/pull/19542)
- Allow overriding to support forwarding incoming headers are forwarded down to target - [PR #19524](https://github.com/BerriAI/litellm/pull/19524)
- **[Chat/Completions](../../docs/completion/input)**
- Add MCP tools response to chat completions - [PR #19552](https://github.com/BerriAI/litellm/pull/19552)
- Add custom vertex ai finish reasons to the output - [PR #19558](https://github.com/BerriAI/litellm/pull/19558)
- Return MCP execution in /chat/completions before model output during streaming - [PR #19623](https://github.com/BerriAI/litellm/pull/19623)
### Bugs
- **[Responses API](../../docs/response_api)**
- Fix duplicate messages during MCP streaming tool execution - [PR #19317](https://github.com/BerriAI/litellm/pull/19317)
- Fix pickle error when using OpenAI's Responses API with stream=True and tool_choice of type allowed_tools (an OpenAI-native parameter) - [PR #17205](https://github.com/BerriAI/litellm/pull/17205)
- stream tool call events for non-openai models - [PR #19368](https://github.com/BerriAI/litellm/pull/19368)
- preserve tool output ordering for gemini in responses bridge - [PR #19360](https://github.com/BerriAI/litellm/pull/19360)
- Add ID caching to prevent ID mismatch text-start and text-delta - [PR #19390](https://github.com/BerriAI/litellm/pull/19390)
- Include output_item, reasoning_summary_Text_done and reasoning_summary_part_done events for non-openai models - [PR #19472](https://github.com/BerriAI/litellm/pull/19472)
- **[Chat/Completions](../../docs/completion/input)**
- fix: drop_params not dropping prompt_cache_key for non-OpenAI providers - [PR #19346](https://github.com/BerriAI/litellm/pull/19346)
- **[Realtime API](../../docs/realtime)**
- disable SSL for ws:// WebSocket connections - [PR #19345](https://github.com/BerriAI/litellm/pull/19345)
- **[Generate Content](../../docs/generateContent)**
- Log actual user input when google genai/vertex endpoints are called client-side - [PR #19156](https://github.com/BerriAI/litellm/pull/19156)
- **[/messages/count_tokens Anthropic Token Counting](../../docs/anthropic_count_tokens)**
- ensure it works for Anthropic, Azure AI Anthropic on AI Gateway - [PR #19432](https://github.com/BerriAI/litellm/pull/19432)
- **[MCP](../../docs/mcp)**
- forward static_headers to MCP servers - [PR #19366](https://github.com/BerriAI/litellm/pull/19366)
- **[Batch API](../../docs/batches)**
- Fix: generation config empty for batch - [PR #19556](https://github.com/BerriAI/litellm/pull/19556)
- **[Pass Through Endpoints](../../docs/proxy/pass_through)**
- Always reupdate registry - [PR #19420](https://github.com/BerriAI/litellm/pull/19420)
---
## Management Endpoints / UI
### Features
- **Cost Estimator**
- Fix model dropdown - [PR #19529](https://github.com/BerriAI/litellm/pull/19529)
- **Claude Code Plugins**
- Allow Adding Claude Code Plugins via UI - [PR #19387](https://github.com/BerriAI/litellm/pull/19387)
- **Guardrails**
- New Policy management UI - [PR #19668](https://github.com/BerriAI/litellm/pull/19668)
- Allow adding policies on Keys/Teams + Viewing on Info panels - [PR #19688](https://github.com/BerriAI/litellm/pull/19688)
- **General**
- respects custom authentication header override - [PR #19276](https://github.com/BerriAI/litellm/pull/19276)
- **Playground**
- Button to Fill Custom API Base - [PR #19440](https://github.com/BerriAI/litellm/pull/19440)
- display mcp output on the play ground - [PR #19553](https://github.com/BerriAI/litellm/pull/19553)
- **Models**
- Paginate /v2/models/info - [PR #19521](https://github.com/BerriAI/litellm/pull/19521)
- All Model Tab Pagination - [PR #19525](https://github.com/BerriAI/litellm/pull/19525)
- Adding Optional scope Param to /models - [PR #19539](https://github.com/BerriAI/litellm/pull/19539)
- Model Search - [PR #19622](https://github.com/BerriAI/litellm/pull/19622)
- Filter by Model ID and Team ID - [PR #19713](https://github.com/BerriAI/litellm/pull/19713)
- **MCP Servers**
- MCP Tools Tab Resetting to Overview - [PR #19468](https://github.com/BerriAI/litellm/pull/19468)
- **Organizations**
- Prevent org admin from creating a new user with proxy_admin permissions - [PR #19296](https://github.com/BerriAI/litellm/pull/19296)
- Edit Page: Reusable Model Select - [PR #19601](https://github.com/BerriAI/litellm/pull/19601)
- **Teams**
- Reusable Model Select - [PR #19543](https://github.com/BerriAI/litellm/pull/19543)
- [Fix] Team Update with Organization having All Proxy Models - [PR #19604](https://github.com/BerriAI/litellm/pull/19604)
- **Logs**
- Include tool arguments in spend logs table - [PR #19640](https://github.com/BerriAI/litellm/pull/19640)
- **Fallbacks / Loadbalancing**
- New fallbacks modal - [PR #19673](https://github.com/BerriAI/litellm/pull/19673)
- Set fallbacks/loadbalancing by team/key - [PR #19686](https://github.com/BerriAI/litellm/pull/19686)
### Bugs
- **Playground**
- increase model selector width in playground Compare view - [PR #19423](https://github.com/BerriAI/litellm/pull/19423)
- **Virtual Keys**
- Sorting Shows Incorrect Entries - [PR #19534](https://github.com/BerriAI/litellm/pull/19534)
- **General**
- UI 404 error when SERVER_ROOT_PATH is set - [PR #19467](https://github.com/BerriAI/litellm/pull/19467)
- Redirect to ui/login on expired JWT - [PR #19687](https://github.com/BerriAI/litellm/pull/19687)
- **SSO**
- Fix SSO user roles not updating for existing users - [PR #19621](https://github.com/BerriAI/litellm/pull/19621)
- **Guardrails**
- ensure guardrail patterns persist on edit and mode toggle - [PR #19265](https://github.com/BerriAI/litellm/pull/19265)
---
## AI Integrations
### Logging
- **General Logging**
- prevent printing duplicate StandardLoggingPayload logs - [PR #19325](https://github.com/BerriAI/litellm/pull/19325)
- Fix: log duplication when json_logs is enabled - [PR #19705](https://github.com/BerriAI/litellm/pull/19705)
- **Langfuse OTEL**
- ignore service logs and fix callback shadowing - [PR #19298](https://github.com/BerriAI/litellm/pull/19298)
- **Langfuse**
- Send litellm_trace_id - [PR #19528](https://github.com/BerriAI/litellm/pull/19528)
- Add Langfuse mock mode for testing without API calls - [PR #19676](https://github.com/BerriAI/litellm/pull/19676)
- **GCS Bucket**
- prevent unbounded queue growth due to slow API calls - [PR #19297](https://github.com/BerriAI/litellm/pull/19297)
- Add GCS mock mode for testing without API calls - [PR #19683](https://github.com/BerriAI/litellm/pull/19683)
- **Responses API Logging**
- Fix pydantic serialization error - [PR #19486](https://github.com/BerriAI/litellm/pull/19486)
- **Arize Phoenix**
- add openinference span kinds to arize phoenix - [PR #19267](https://github.com/BerriAI/litellm/pull/19267)
- **Prometheus**
- Added new prometheus metrics for user count and team count - [PR #19520](https://github.com/BerriAI/litellm/pull/19520)
### Guardrails
- **Bedrock Guardrails**
- Ensure post_call guardrail checks input+output - [PR #19151](https://github.com/BerriAI/litellm/pull/19151)
- **Prompt Security**
- fixing prompt-security's guardrail implementation - [PR #19374](https://github.com/BerriAI/litellm/pull/19374)
- **Presidio**
- Fixes crash in Presidio Guardrail when running in background threads (logging_hook) - [PR #19714](https://github.com/BerriAI/litellm/pull/19714)
- **Pillar Security**
- Migrate Pillar Security to Generic Guardrail API - [PR #19364](https://github.com/BerriAI/litellm/pull/19364)
- **Policy Engine**
- New LiteLLM Policy engine - create policies to manage guardrails, conditions - permissions per Key, Team - [PR #19612](https://github.com/BerriAI/litellm/pull/19612)
- **General**
- add case-insensitive support for guardrail mode and actions - [PR #19480](https://github.com/BerriAI/litellm/pull/19480)
### Prompt Management
- **General**
- fix prompt info lookup and delete using correct IDs - [PR #19358](https://github.com/BerriAI/litellm/pull/19358)
### Secret Manager
- **AWS Secret Manager**
- ensure auto-rotation updates existing AWS secret instead of creating new one - [PR #19455](https://github.com/BerriAI/litellm/pull/19455)
- **Hashicorp Vault**
- Ensure key rotations work with Vault - [PR #19634](https://github.com/BerriAI/litellm/pull/19634)
---
## Spend Tracking, Budgets and Rate Limiting
- **Pricing Updates**
- Add openai/dall-e base pricing entries - [PR #19133](https://github.com/BerriAI/litellm/pull/19133)
- Add `input_cost_per_video_per_second` in ModelInfoBase - [PR #19398](https://github.com/BerriAI/litellm/pull/19398)
---
## Performance / Loadbalancing / Reliability improvements
- **General**
- Fix date overflow/division by zero in proxy utils - [PR #19527](https://github.com/BerriAI/litellm/pull/19527)
- Fix in-flight request termination on SIGTERM when health-check runs in a separate process - [PR #19427](https://github.com/BerriAI/litellm/pull/19427)
- Fix Pass through routes to work with server root path - [PR #19383](https://github.com/BerriAI/litellm/pull/19383)
- Fix logging error for stop iteration - [PR #19649](https://github.com/BerriAI/litellm/pull/19649)
- prevent retrying 4xx client errors - [PR #19275](https://github.com/BerriAI/litellm/pull/19275)
- add better error handling for misconfig on health check - [PR #19441](https://github.com/BerriAI/litellm/pull/19441)
- **Router**
- Fix Azure RPM calculation formula - [PR #19513](https://github.com/BerriAI/litellm/pull/19513)
- Persist scheduler request queue to redis - [PR #19304](https://github.com/BerriAI/litellm/pull/19304)
- pass search_tools to Router during DB-triggered initialization - [PR #19388](https://github.com/BerriAI/litellm/pull/19388)
- Fixed PromptCachingCache to correctly handle messages where cache_control is a sibling key of string content - [PR #19266](https://github.com/BerriAI/litellm/pull/19266)
- **Memory Leaks/OOM**
- prevent OOM with nested $defs in tool schemas - [PR #19112](https://github.com/BerriAI/litellm/pull/19112)
- fix: HTTP client memory leaks in Presidio, OpenAI, and Gemini - [PR #19190](https://github.com/BerriAI/litellm/pull/19190)
- **Non root**
- fix logfile and pidfile of supervisor for non root environment - [PR #17267](https://github.com/BerriAI/litellm/pull/17267)
- resolve Read-only file system error in non-root images - [PR #19449](https://github.com/BerriAI/litellm/pull/19449)
- **Dockerfile**
- Redis Semantic Caching - add missing redisvl dependency to requirements.txt - [PR #19417](https://github.com/BerriAI/litellm/pull/19417)
- Bump OTEL versions to support a2a dependency - resolves modulenotfounderror for Microsoft Agents by @Harshit28j in #18991
- **DB**
- Handle PostgreSQL cached plan errors during rolling deployments - [PR #19424](https://github.com/BerriAI/litellm/pull/19424)
- **Timeouts**
- Fix: total timeout is not respected - [PR #19389](https://github.com/BerriAI/litellm/pull/19389)
- **SDK**
- Field-Existence Checks to Type Classes to Prevent Attribute Errors - [PR #18321](https://github.com/BerriAI/litellm/pull/18321)
- add google-cloud-aiplatform as optional dependency with clear error message - [PR #19437](https://github.com/BerriAI/litellm/pull/19437)
- Make grpc dependency optional - [PR #19447](https://github.com/BerriAI/litellm/pull/19447)
- Add support for retry policies - [PR #19645](https://github.com/BerriAI/litellm/pull/19645)
- **Performance**
- Cut chat_completion latency by ~21% by reducing pre-call processing time - [PR #19535](https://github.com/BerriAI/litellm/pull/19535)
- Optimize strip_trailing_slash with O(1) index check - [PR #19679](https://github.com/BerriAI/litellm/pull/19679)
- Optimize use_custom_pricing_for_model with set intersection - [PR #19677](https://github.com/BerriAI/litellm/pull/19677)
- perf: skip pattern_router.route() for non-wildcard models - [PR #19664](https://github.com/BerriAI/litellm/pull/19664)
- perf: Add LRU caching to get_model_info for faster cost lookups - [PR #19606](https://github.com/BerriAI/litellm/pull/19606)
---
## General Proxy Improvements
### Doc Improvements
- new tutorial for adding MCPs to Cursor via LiteLLM - [PR #19317](https://github.com/BerriAI/litellm/pull/19317)
- fix vertex_region to vertex_location in Vertex AI pass-through docs - [PR #19380](https://github.com/BerriAI/litellm/pull/19380)
- clarify Gemini and Vertex AI model prefix in json file - [PR #19443](https://github.com/BerriAI/litellm/pull/19443)
- update Claude Code integration guides - [PR #19415](https://github.com/BerriAI/litellm/pull/19415)
- adjust opencode tutorial - [PR #19605](https://github.com/BerriAI/litellm/pull/19605)
- add spend-queue-troubleshooting docs - [PR #19659](https://github.com/BerriAI/litellm/pull/19659)
- docs: add litellm-enterprise requirement for managed files - [PR #19689](https://github.com/BerriAI/litellm/pull/19689)
### Helm
- Add support for keda in helm chart - [PR #19337](https://github.com/BerriAI/litellm/pull/19337)
- sync Helm chart version with LiteLLM release version - [PR #19438](https://github.com/BerriAI/litellm/pull/19438)
- Enable PreStop hook configuration in values.yaml - [PR #19613](https://github.com/BerriAI/litellm/pull/19613)
### General
- Add health check scripts and parallel execution support - [PR #19295](https://github.com/BerriAI/litellm/pull/19295)
---
## New Contributors
* @dushyantzz made their first contribution in [PR #19158](https://github.com/BerriAI/litellm/pull/19158)
* @obod-mpw made their first contribution in [PR #19133](https://github.com/BerriAI/litellm/pull/19133)
* @msexxeta made their first contribution in [PR #19030](https://github.com/BerriAI/litellm/pull/19030)
* @rsicart made their first contribution in [PR #19337](https://github.com/BerriAI/litellm/pull/19337)
* @cluebbehusen made their first contribution in [PR #19311](https://github.com/BerriAI/litellm/pull/19311)
* @Lucky-Lodhi2004 made their first contribution in [PR #19315](https://github.com/BerriAI/litellm/pull/19315)
* @binbandit made their first contribution in [PR #19324](https://github.com/BerriAI/litellm/pull/19324)
* @flex-myeonghyeon made their first contribution in [PR #19381](https://github.com/BerriAI/litellm/pull/19381)
* @Lrakotoson made their first contribution in [PR #18321](https://github.com/BerriAI/litellm/pull/18321)
* @bensi94 made their first contribution in [PR #18787](https://github.com/BerriAI/litellm/pull/18787)
* @victorigualada made their first contribution in [PR #19368](https://github.com/BerriAI/litellm/pull/19368)
* @VedantMadane made their first contribution in #19266
* @stiyyagura0901 made their first contribution in #19276
* @kamilio made their first contribution in [PR #19447](https://github.com/BerriAI/litellm/pull/19447)
* @jonathansampson made their first contribution in [PR #19433](https://github.com/BerriAI/litellm/pull/19433)
* @rynecarbone made their first contribution in [PR #19416](https://github.com/BerriAI/litellm/pull/19416)
* @jayy-77 made their first contribution in #19366
* @davida-ps made their first contribution in [PR #19374](https://github.com/BerriAI/litellm/pull/19374)
* @joaodinissf made their first contribution in [PR #19506](https://github.com/BerriAI/litellm/pull/19506)
* @ecao310 made their first contribution in [PR #19520](https://github.com/BerriAI/litellm/pull/19520)
* @mpcusack-altos made their first contribution in [PR #19577](https://github.com/BerriAI/litellm/pull/19577)
* @milan-berri made their first contribution in [PR #19602](https://github.com/BerriAI/litellm/pull/19602)
* @xqe2011 made their first contribution in #19621
---
## Full Changelog
**[View complete changelog on GitHub](https://github.com/BerriAI/litellm/releases/tag/v1.81.3.rc)**

384
docs/my-website/release_notes/v1.81.6.md Normal file
View file

@ -0,0 +1,384 @@
---
title: "v1.81.6 - Logs v2 with Tool Call Tracing"
slug: "v1-81-6"
date: 2026-01-31T00:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
## Deploy this version
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
<Tabs>
<TabItem value="docker" label="Docker">
```bash
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:main-v1.81.6
```
</TabItem>
<TabItem value="pip" label="Pip">
```bash
pip install litellm==1.81.6
```
</TabItem>
</Tabs>
## Key Highlights
Logs View v2 with Tool Call Tracing - Redesigned logs interface with side panel, structured tool visualization, and error message search for faster debugging.
Let's dive in.
### Logs View v2 with Tool Call Tracing
This release introduces comprehensive tool call tracing through LiteLLM's redesigned Logs View v2, enabling developers to debug and monitor AI agent workflows in production environments seamlessly.
This means you can now onboard use cases like tracing complex multi-step agent interactions, debugging tool execution failures, and monitoring MCP server calls while maintaining full visibility into request/response payloads with syntax highlighting.
Developers can access the new Logs View through LiteLLM's UI to inspect tool calls in structured format, search logs by error messages or request patterns, and correlate agent activities across sessions with collapsible side panel views.
{/* TODO: Add image from Slack (group_7219.png) - save as logs_v2_tool_tracing.png */}
{/* <Image img={require('../../img/release_notes/logs_v2_tool_tracing.png')} style={{ maxWidth: '800px', width: '100%' }} /> */}
[Get Started](../../docs/proxy/ui_logs)
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| AWS Bedrock | `amazon.nova-2-pro-preview-20251202-v1:0` | 1M | $2.19 | $17.50 | Chat completions, vision, video, PDF, function calling, prompt caching, reasoning |
| Google Vertex AI | `gemini-robotics-er-1.5-preview` | 1M | $0.30 | $2.50 | Chat completions, multimodal (text, image, video, audio), function calling, reasoning |
| OpenRouter | `openrouter/xiaomi/mimo-v2-flash` | 262K | $0.09 | $0.29 | Chat completions, function calling, reasoning |
| OpenRouter | `openrouter/moonshotai/kimi-k2.5` | - | - | - | Chat completions |
| OpenRouter | `openrouter/z-ai/glm-4.7` | 202K | $0.40 | $1.50 | Chat completions, vision, function calling, reasoning |
#### Features
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Messages API Bedrock Converse caching and PDF support - [PR #19785](https://github.com/BerriAI/litellm/pull/19785)
- Translate advanced-tool-use to Bedrock-specific headers for Claude Opus 4.5 - [PR #19841](https://github.com/BerriAI/litellm/pull/19841)
- Support tool search header translation for Sonnet 4.5 - [PR #19871](https://github.com/BerriAI/litellm/pull/19871)
- Filter unsupported beta headers for AWS Bedrock Invoke API - [PR #19877](https://github.com/BerriAI/litellm/pull/19877)
- Nova grounding improvements - [PR #19598](https://github.com/BerriAI/litellm/pull/19598), [PR #20159](https://github.com/BerriAI/litellm/pull/20159)
- **[Anthropic](../../docs/providers/anthropic)**
- Remove explicit cache_control null in tool_result content - [PR #19919](https://github.com/BerriAI/litellm/pull/19919)
- Fix tool handling - [PR #19805](https://github.com/BerriAI/litellm/pull/19805)
- **[Google Gemini / Vertex AI](../../docs/providers/gemini)**
- Add Gemini Robotics-ER 1.5 preview support - [PR #19845](https://github.com/BerriAI/litellm/pull/19845)
- Support file retrieval in GoogleAIStudioFilesHandle - [PR #20018](https://github.com/BerriAI/litellm/pull/20018)
- Add /delete endpoint support - [PR #20055](https://github.com/BerriAI/litellm/pull/20055)
- Add custom_llm_provider as gemini translation - [PR #19988](https://github.com/BerriAI/litellm/pull/19988)
- Subtract implicit cached tokens from text_tokens for correct cost calculation - [PR #19775](https://github.com/BerriAI/litellm/pull/19775)
- Remove unsupported prompt-caching-scope-2026-01-05 header for vertex ai - [PR #20058](https://github.com/BerriAI/litellm/pull/20058)
- Add disable flag for anthropic gemini cache translation - [PR #20052](https://github.com/BerriAI/litellm/pull/20052)
- Convert image URLs to base64 in tool messages for Anthropic on Vertex AI - [PR #19896](https://github.com/BerriAI/litellm/pull/19896)
- **[xAI](../../docs/providers/xai)**
- Add grok reasoning content support - [PR #19850](https://github.com/BerriAI/litellm/pull/19850)
- Add websearch params support for Responses API - [PR #19915](https://github.com/BerriAI/litellm/pull/19915)
- Add routing of xai chat completions to responses when web search options is present - [PR #20051](https://github.com/BerriAI/litellm/pull/20051)
- Correct cached token cost calculation - [PR #19772](https://github.com/BerriAI/litellm/pull/19772)
- **[Azure OpenAI](../../docs/providers/azure)**
- Use generic cost calculator for audio token pricing - [PR #19771](https://github.com/BerriAI/litellm/pull/19771)
- Allow tool_choice for Azure GPT-5 chat models - [PR #19813](https://github.com/BerriAI/litellm/pull/19813)
- Set gpt-5.2-codex mode to responses for Azure and OpenRouter - [PR #19770](https://github.com/BerriAI/litellm/pull/19770)
- **[OpenAI](../../docs/providers/openai)**
- Fix max_input_tokens for gpt-5.2-codex - [PR #20009](https://github.com/BerriAI/litellm/pull/20009)
- Fix gpt-image-1.5 cost calculation not including output image tokens - [PR #19515](https://github.com/BerriAI/litellm/pull/19515)
- **[Hosted VLLM](../../docs/providers/vllm)**
- Support thinking parameter in anthropic_messages() and .completion() - [PR #19787](https://github.com/BerriAI/litellm/pull/19787)
- Route through base_llm_http_handler to support ssl_verify - [PR #19893](https://github.com/BerriAI/litellm/pull/19893)
- Fix vllm embedding format - [PR #20056](https://github.com/BerriAI/litellm/pull/20056)
- **[OCI GenAI](../../docs/providers/oci)**
- Serialize imageUrl as object for OCI GenAI API - [PR #19661](https://github.com/BerriAI/litellm/pull/19661)
- **[Volcengine](../../docs/providers/volcano)**
- Add context for volcengine models (deepseek-v3-2, glm-4-7, kimi-k2-thinking) - [PR #19335](https://github.com/BerriAI/litellm/pull/19335)
- **[Chinese Providers](../../docs/providers/)**
- Add prompt caching and reasoning support for MiniMax, GLM, Xiaomi - [PR #19924](https://github.com/BerriAI/litellm/pull/19924)
- **[Vercel AI Gateway](../../docs/providers/vercel_ai_gateway)**
- Add embeddings support - [PR #19660](https://github.com/BerriAI/litellm/pull/19660)
### Bug Fixes
- **[Google](../../docs/providers/gemini)**
- Fix gemini-robotics-er-1.5-preview entry - [PR #19974](https://github.com/BerriAI/litellm/pull/19974)
- **General**
- Fix output_tokens_details.reasoning_tokens None - [PR #19914](https://github.com/BerriAI/litellm/pull/19914)
- Fix stream_chunk_builder to preserve images from streaming chunks - [PR #19654](https://github.com/BerriAI/litellm/pull/19654)
- Fix aspectRatio mapping in image edit - [PR #20053](https://github.com/BerriAI/litellm/pull/20053)
- Handle unknown models in Azure AI cost calculator - [PR #20150](https://github.com/BerriAI/litellm/pull/20150)
- **[GigaChat](../../docs/providers/gigachat)**
- Ensure function content is valid JSON - [PR #19232](https://github.com/BerriAI/litellm/pull/19232)
## LLM API Endpoints
#### Features
- **[Messages API (/messages)](../../docs/mcp)**
- Add LiteLLM x Claude Agent SDK Integration - [PR #20035](https://github.com/BerriAI/litellm/pull/20035)
- **[A2A / MCP Gateway API (/a2a, /mcp)](../../docs/mcp)**
- Add A2A agent header-based context propagation support - [PR #19504](https://github.com/BerriAI/litellm/pull/19504)
- Enable progress notifications for MCP tool calls - [PR #19809](https://github.com/BerriAI/litellm/pull/19809)
- Fix support for non-standard MCP URL patterns - [PR #19738](https://github.com/BerriAI/litellm/pull/19738)
- Add backward compatibility for legacy A2A card formats (/.well-known/agent.json) - [PR #19949](https://github.com/BerriAI/litellm/pull/19949)
- Add support for agent parameter in /interactions endpoint - [PR #19866](https://github.com/BerriAI/litellm/pull/19866)
- **[Responses API (/responses)](../../docs/response_api)**
- Fix custom_llm_provider for provider-specific params - [PR #19798](https://github.com/BerriAI/litellm/pull/19798)
- Extract input tokens details as dict in ResponseAPILoggingUtils - [PR #20046](https://github.com/BerriAI/litellm/pull/20046)
- **[Batch API (/batches)](../../docs/batches)**
- Fix /batches to return encoded ids (from managed objects table) - [PR #19040](https://github.com/BerriAI/litellm/pull/19040)
- Fix Batch and File user level permissions - [PR #19981](https://github.com/BerriAI/litellm/pull/19981)
- Add cost tracking and usage object in retrieve_batch call type - [PR #19986](https://github.com/BerriAI/litellm/pull/19986)
- **[Embeddings API (/embeddings)](../../docs/embedding/supported_embedding)**
- Add supported input formats documentation - [PR #20073](https://github.com/BerriAI/litellm/pull/20073)
- **[RAG API (/rag/ingest, /vector_store)](../../docs/rag_ingest)**
- Add UI for /rag/ingest API - Upload docs, pdfs etc to create vector stores - [PR #19822](https://github.com/BerriAI/litellm/pull/19822)
- Add support for using S3 Vectors as Vector Store Provider - [PR #19888](https://github.com/BerriAI/litellm/pull/19888)
- Add s3_vectors as provider on /vector_store/search API + UI for creating + PDF support - [PR #19895](https://github.com/BerriAI/litellm/pull/19895)
- Add permission management for users and teams on Vector Stores - [PR #19972](https://github.com/BerriAI/litellm/pull/19972)
- Enable router support for completions in RAG query pipeline - [PR #19550](https://github.com/BerriAI/litellm/pull/19550)
- **[Search API (/search)](../../docs/search)**
- Add /list endpoint to list what search tools exist in router - [PR #19969](https://github.com/BerriAI/litellm/pull/19969)
- Fix router search tools v2 integration - [PR #19840](https://github.com/BerriAI/litellm/pull/19840)
- **[Passthrough Endpoints (/\{provider\}_passthrough)](../../docs/pass_through/intro)**
- Add /openai_passthrough route for OpenAI passthrough requests - [PR #19989](https://github.com/BerriAI/litellm/pull/19989)
- Add support for configuring role_mappings via environment variables - [PR #19498](https://github.com/BerriAI/litellm/pull/19498)
- Add Vertex AI LLM credentials sensitive keyword "vertex_credentials" for masking - [PR #19551](https://github.com/BerriAI/litellm/pull/19551)
- Fix prevention of provider-prefixed model name leaks in responses - [PR #19943](https://github.com/BerriAI/litellm/pull/19943)
- Fix proxy support for slashes in Google Vertex generateContent model names - [PR #19737](https://github.com/BerriAI/litellm/pull/19737), [PR #19753](https://github.com/BerriAI/litellm/pull/19753)
- Support model names with slashes in Vertex AI passthrough URLs - [PR #19944](https://github.com/BerriAI/litellm/pull/19944)
- Fix regression in Vertex AI passthroughs for router models - [PR #19967](https://github.com/BerriAI/litellm/pull/19967)
- Add regression tests for Vertex AI passthrough model names - [PR #19855](https://github.com/BerriAI/litellm/pull/19855)
#### Bugs
- **General**
- Fix token calculations and refactor - [PR #19696](https://github.com/BerriAI/litellm/pull/19696)
## Management Endpoints / UI
#### Features
- **Proxy CLI Auth**
- Add configurable CLI JWT expiration via environment variable - [PR #19780](https://github.com/BerriAI/litellm/pull/19780)
- Fix team cli auth flow - [PR #19666](https://github.com/BerriAI/litellm/pull/19666)
- **Virtual Keys**
- UI: Auto Truncation of Table Values - [PR #19718](https://github.com/BerriAI/litellm/pull/19718)
- Fix Create Key: Expire Key Input Duration - [PR #19807](https://github.com/BerriAI/litellm/pull/19807)
- Bulk Update Keys Endpoint - [PR #19886](https://github.com/BerriAI/litellm/pull/19886)
- **Logs View**
- **v2 Logs view with side panel and improved UX** - [PR #20091](https://github.com/BerriAI/litellm/pull/20091)
- New View to render "Tools" on Logs View - [PR #20093](https://github.com/BerriAI/litellm/pull/20093)
- Add Pretty print view of request/response - [PR #20096](https://github.com/BerriAI/litellm/pull/20096)
- Add error_message search in Spend Logs Endpoint - [PR #19960](https://github.com/BerriAI/litellm/pull/19960)
- UI: Adding Error message search to ui spend logs - [PR #19963](https://github.com/BerriAI/litellm/pull/19963)
- Spend Logs: Settings Modal - [PR #19918](https://github.com/BerriAI/litellm/pull/19918)
- Fix error_code in Spend Logs metadata - [PR #20015](https://github.com/BerriAI/litellm/pull/20015)
- Spend Logs: Show Current Store and Retention Status - [PR #20017](https://github.com/BerriAI/litellm/pull/20017)
- Allow Dynamic Setting of store_prompts_in_spend_logs - [PR #19913](https://github.com/BerriAI/litellm/pull/19913)
- [Docs: UI Spend Logs Settings](../../docs/proxy/ui_spend_log_settings) - [PR #20197](https://github.com/BerriAI/litellm/pull/20197)
- **Models + Endpoints**
- Add sortBy and sortOrder params for /v2/model/info - [PR #19903](https://github.com/BerriAI/litellm/pull/19903)
- Fix Sorting for /v2/model/info - [PR #19971](https://github.com/BerriAI/litellm/pull/19971)
- UI: Model Page Server Sort - [PR #19908](https://github.com/BerriAI/litellm/pull/19908)
- **Usage & Analytics**
- UI: Usage Export: Breakdown by Teams and Keys - [PR #19953](https://github.com/BerriAI/litellm/pull/19953)
- UI: Usage: Model Breakdown Per Key - [PR #20039](https://github.com/BerriAI/litellm/pull/20039)
- **UI Improvements**
- UI: Allow Admins to control what pages are visible on LeftNav - [PR #19907](https://github.com/BerriAI/litellm/pull/19907)
- UI: Add Light/Dark Mode Switch for Development - [PR #19804](https://github.com/BerriAI/litellm/pull/19804)
- UI: Dark Mode: Delete Resource Modal - [PR #20098](https://github.com/BerriAI/litellm/pull/20098)
- UI: Tables: Reusable Table Sort Component - [PR #19970](https://github.com/BerriAI/litellm/pull/19970)
- UI: New Badge Dot Render - [PR #20024](https://github.com/BerriAI/litellm/pull/20024)
- UI: Feedback Prompts: Option To Hide Prompts - [PR #19831](https://github.com/BerriAI/litellm/pull/19831)
- UI: Navbar: Fixed Default Logo + Bound Logo Box - [PR #20092](https://github.com/BerriAI/litellm/pull/20092)
- UI: Navbar: User Dropdown - [PR #20095](https://github.com/BerriAI/litellm/pull/20095)
- Change default key type from 'Default' to 'LLM API' - [PR #19516](https://github.com/BerriAI/litellm/pull/19516)
- **Team & User Management**
- Fix /team/member_add User Email and ID Verifications - [PR #19814](https://github.com/BerriAI/litellm/pull/19814)
- Fix SSO Email Case Sensitivity - [PR #19799](https://github.com/BerriAI/litellm/pull/19799)
- UI: Internal User: Bulk Add - [PR #19721](https://github.com/BerriAI/litellm/pull/19721)
- **AI Gateway Features**
- Add support for making silent LLM calls without logging - [PR #19544](https://github.com/BerriAI/litellm/pull/19544)
- UI: Fix MCP tools instructions to display comma-separated strings - [PR #20101](https://github.com/BerriAI/litellm/pull/20101)
#### Bugs
- Fix Model Name During Fallback - [PR #20177](https://github.com/BerriAI/litellm/pull/20177)
- Fix Health Endpoints when Callback Objects Defined - [PR #20182](https://github.com/BerriAI/litellm/pull/20182)
- Fix Unable to reset user max budget to unlimited - [PR #19796](https://github.com/BerriAI/litellm/pull/19796)
- Fix Password comparison with non-ASCII characters - [PR #19568](https://github.com/BerriAI/litellm/pull/19568)
- Correct error message for DISABLE_ADMIN_ENDPOINTS - [PR #19861](https://github.com/BerriAI/litellm/pull/19861)
- Prevent clearing content filter patterns when editing guardrail - [PR #19671](https://github.com/BerriAI/litellm/pull/19671)
- Fix Prompt Studio history to load tools and system messages - [PR #19920](https://github.com/BerriAI/litellm/pull/19920)
- Add WATSONX_ZENAPIKEY to WatsonX credentials - [PR #20086](https://github.com/BerriAI/litellm/pull/20086)
- UI: Vector Store: Allow Config Defined Models to Be Selected - [PR #20031](https://github.com/BerriAI/litellm/pull/20031)
## Logging / Guardrail / Prompt Management Integrations
#### Features
- **[DataDog](../../docs/proxy/logging#datadog)**
- Add agent support for LLM Observability - [PR #19574](https://github.com/BerriAI/litellm/pull/19574)
- Add datadog cost management support and fix startup callback issue - [PR #19584](https://github.com/BerriAI/litellm/pull/19584)
- Add datadog_llm_observability to /health/services allowed list - [PR #19952](https://github.com/BerriAI/litellm/pull/19952)
- Check for agent mode before requiring DD_API_KEY/DD_SITE - [PR #20156](https://github.com/BerriAI/litellm/pull/20156)
- **[OpenTelemetry](../../docs/observability/opentelemetry_integration)**
- Propagate JWT auth metadata to OTEL spans - [PR #19627](https://github.com/BerriAI/litellm/pull/19627)
- Fix thread leak in dynamic header path - [PR #19946](https://github.com/BerriAI/litellm/pull/19946)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Add callbacks and labels - [PR #19708](https://github.com/BerriAI/litellm/pull/19708)
- Add clientip and user agent in metrics - [PR #19717](https://github.com/BerriAI/litellm/pull/19717)
- Add tpm-rpm limit metrics - [PR #19725](https://github.com/BerriAI/litellm/pull/19725)
- Add model_id label to metrics - [PR #19678](https://github.com/BerriAI/litellm/pull/19678)
- Safely handle None metadata in logging - [PR #19691](https://github.com/BerriAI/litellm/pull/19691)
- Resolve high CPU when router_settings in DB by avoiding REGISTRY.collect() - [PR #20087](https://github.com/BerriAI/litellm/pull/20087)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Add litellm_callback_logging_failures_metric for Langfuse, Langfuse Otel and other Otel providers - [PR #19636](https://github.com/BerriAI/litellm/pull/19636)
- **General Logging**
- Use return value from CustomLogger.async_post_call_success_hook - [PR #19670](https://github.com/BerriAI/litellm/pull/19670)
- Add async_post_call_response_headers_hook to CustomLogger - [PR #20083](https://github.com/BerriAI/litellm/pull/20083)
- Add mock client factory pattern and mock support for PostHog, Helicone, and Braintrust integrations - [PR #19707](https://github.com/BerriAI/litellm/pull/19707)
#### Guardrails
- **[Presidio](../../docs/proxy/guardrails/pii_masking_v2)**
- Reuse HTTP connections to prevent performance degradation - [PR #19964](https://github.com/BerriAI/litellm/pull/19964)
- **Onyx**
- Add timeout to onyx guardrail - [PR #19731](https://github.com/BerriAI/litellm/pull/19731)
- **General**
- Add guardrail model argument feature - [PR #19619](https://github.com/BerriAI/litellm/pull/19619)
- Fix guardrails issues with streaming-response regex - [PR #19901](https://github.com/BerriAI/litellm/pull/19901)
- Remove enterprise requirement for guardrail monitoring (docs) - [PR #19833](https://github.com/BerriAI/litellm/pull/19833)
## Spend Tracking, Budgets and Rate Limiting
- Add event-driven coordination for global spend query to prevent cache stampede - [PR #20030](https://github.com/BerriAI/litellm/pull/20030)
## Performance / Loadbalancing / Reliability improvements
- **Resolve high CPU when router_settings in DB** - by avoiding REGISTRY.collect() in PrometheusServicesLogger - [PR #20087](https://github.com/BerriAI/litellm/pull/20087)
- **Reuse HTTP connections in Presidio** - to prevent performance degradation - [PR #19964](https://github.com/BerriAI/litellm/pull/19964)
- **Event-driven coordination for global spend query** - prevent cache stampede - [PR #20030](https://github.com/BerriAI/litellm/pull/20030)
- Fix recursive Pydantic validation issue - [PR #19531](https://github.com/BerriAI/litellm/pull/19531)
- Refactor argument handling into helper function to reduce code bloat - [PR #19720](https://github.com/BerriAI/litellm/pull/19720)
- Optimize logo fetching and resolve MCP import blockers - [PR #19719](https://github.com/BerriAI/litellm/pull/19719)
- Improve logo download performance using async HTTP client - [PR #20155](https://github.com/BerriAI/litellm/pull/20155)
- Fix server root path configuration - [PR #19790](https://github.com/BerriAI/litellm/pull/19790)
- Refactor: Extract transport context creation into separate method - [PR #19794](https://github.com/BerriAI/litellm/pull/19794)
- Add native_background_mode configuration to override polling_via_cache for specific models - [PR #19899](https://github.com/BerriAI/litellm/pull/19899)
- Initialize tiktoken environment at import time to enable offline usage - [PR #19882](https://github.com/BerriAI/litellm/pull/19882)
- Improve tiktoken performance using local cache in lazy loading - [PR #19774](https://github.com/BerriAI/litellm/pull/19774)
- Fix timeout errors in chat completion calls to be correctly reported in failure callbacks - [PR #19842](https://github.com/BerriAI/litellm/pull/19842)
- Fix environment variable type handling for NUM_RETRIES - [PR #19507](https://github.com/BerriAI/litellm/pull/19507)
- Use safe_deep_copy in silent experiment kwargs to prevent mutation - [PR #20170](https://github.com/BerriAI/litellm/pull/20170)
- Improve error handling by inspecting BadRequestError after all other policy types - [PR #19878](https://github.com/BerriAI/litellm/pull/19878)
## Database Changes
### Schema Updates
| Table | Change Type | Description | PR | Migration |
| ----- | ----------- | ----------- | -- | --------- |
| `LiteLLM_ManagedVectorStoresTable` | New Columns | Added `team_id` and `user_id` fields for permission management | [PR #19972](https://github.com/BerriAI/litellm/pull/19972) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260131150814_add_team_user_to_vector_stores/migration.sql) |
### Migration Improvements
- Fix Docker: Use correct schema path for Prisma generation - [PR #19631](https://github.com/BerriAI/litellm/pull/19631)
- Resolve 'relation does not exist' migration errors in setup_database - [PR #19281](https://github.com/BerriAI/litellm/pull/19281)
- Fix migration issue and improve Docker image stability - [PR #19843](https://github.com/BerriAI/litellm/pull/19843)
- Run Prisma generate as nobody user in non-root Docker container for security - [PR #20000](https://github.com/BerriAI/litellm/pull/20000)
- Bump litellm-proxy-extras version to 0.4.28 - [PR #20166](https://github.com/BerriAI/litellm/pull/20166)
## Documentation Updates
- **[Add Claude Agents SDK x LiteLLM Guide](../../docs/mcp)** - [PR #20036](https://github.com/BerriAI/litellm/pull/20036)
- **[Add Cookbook: Using Claude Agent SDK + MCPs with LiteLLM](https://github.com/BerriAI/litellm/tree/main/cookbook)** - [PR #20081](https://github.com/BerriAI/litellm/pull/20081)
- Fix A2A Python SDK URL in documentation - [PR #19832](https://github.com/BerriAI/litellm/pull/19832)
- **[Add Sarvam usage documentation](../../docs/providers/sarvam)** - [PR #19844](https://github.com/BerriAI/litellm/pull/19844)
- **[Add supported input formats for embeddings](../../docs/embedding/supported_embedding)** - [PR #20073](https://github.com/BerriAI/litellm/pull/20073)
- **[UI Spend Logs Settings Docs](../../docs/proxy/ui_spend_log_settings)** - [PR #20197](https://github.com/BerriAI/litellm/pull/20197)
- Add OpenAI Agents SDK to OSS Adopters list in README - [PR #19820](https://github.com/BerriAI/litellm/pull/19820)
- Update docs: Remove enterprise requirement for guardrail monitoring - [PR #19833](https://github.com/BerriAI/litellm/pull/19833)
- Add missing environment variable documentation - [PR #20138](https://github.com/BerriAI/litellm/pull/20138)
- Improve documentation blog index page - [PR #20188](https://github.com/BerriAI/litellm/pull/20188)
## Infrastructure / Testing Improvements
- Add test coverage for Router.get_valid_args and improve code coverage reporting - [PR #19797](https://github.com/BerriAI/litellm/pull/19797)
- Add validation of model cost map as CI job - [PR #19993](https://github.com/BerriAI/litellm/pull/19993)
- Add Realtime API benchmarks - [PR #20074](https://github.com/BerriAI/litellm/pull/20074)
- Add Init Containers support in community helm chart - [PR #19816](https://github.com/BerriAI/litellm/pull/19816)
- Add libsndfile to main Dockerfile for ARM64 audio processing support - [PR #19776](https://github.com/BerriAI/litellm/pull/19776)
## New Contributors
* @ruanjf made their first contribution in https://github.com/BerriAI/litellm/pull/19551
* @moh-dev-stack made their first contribution in https://github.com/BerriAI/litellm/pull/19507
* @formorter made their first contribution in https://github.com/BerriAI/litellm/pull/19498
* @priyam-that made their first contribution in https://github.com/BerriAI/litellm/pull/19516
* @marcosgriselli made their first contribution in https://github.com/BerriAI/litellm/pull/19550
* @natimofeev made their first contribution in https://github.com/BerriAI/litellm/pull/19232
* @zifeo made their first contribution in https://github.com/BerriAI/litellm/pull/19805
* @pragyasardana made their first contribution in https://github.com/BerriAI/litellm/pull/19816
* @ryewilson made their first contribution in https://github.com/BerriAI/litellm/pull/19833
* @lizhen921 made their first contribution in https://github.com/BerriAI/litellm/pull/19919
* @boarder7395 made their first contribution in https://github.com/BerriAI/litellm/pull/19666
* @rushilchugh01 made their first contribution in https://github.com/BerriAI/litellm/pull/19938
* @cfchase made their first contribution in https://github.com/BerriAI/litellm/pull/19893
* @ayim made their first contribution in https://github.com/BerriAI/litellm/pull/19872
* @varunsripad123 made their first contribution in https://github.com/BerriAI/litellm/pull/20018
* @nht1206 made their first contribution in https://github.com/BerriAI/litellm/pull/20046
* @genga6 made their first contribution in https://github.com/BerriAI/litellm/pull/20009
**Full Changelog**: https://github.com/BerriAI/litellm/compare/v1.81.3.rc...v1.81.6

49
docs/my-website/sidebars.js
View file

@ -79,6 +79,7 @@ const sidebars = {
"proxy/guardrails/panw_prisma_airs",
"proxy/guardrails/secret_detection",
"proxy/guardrails/custom_guardrail",
"proxy/guardrails/custom_code_guardrail",
"proxy/guardrails/prompt_injection",
"proxy/guardrails/tool_permission",
"proxy/guardrails/zscaler_ai_guard",
@ -139,6 +140,22 @@ const sidebars = {
"tutorials/openai_codex"
]
},
{
type: "category",
label: "Agent SDKs",
link: {
type: "generated-index",
title: "Agent SDKs",
description: "Use LiteLLM with agent frameworks and SDKs",
slug: "/agent_sdks"
},
items: [
"tutorials/claude_agent_sdk",
"tutorials/copilotkit_sdk",
"tutorials/google_adk",
"tutorials/livekit_xai_realtime",
]
},
],
// But you can create a sidebar manually
@ -274,11 +291,19 @@ const sidebars = {
"proxy/custom_sso",
"proxy/ai_hub",
"proxy/model_compare_ui",
"proxy/public_teams",
"proxy/self_serve",
"proxy/ui/bulk_edit_users",
"proxy/ui_credentials",
"tutorials/scim_litellm",
{
type: "category",
label: "UI User/Team Management",
items: [
"proxy/access_control",
"proxy/public_teams",
"proxy/self_serve",
"proxy/ui/bulk_edit_users",
"proxy/ui/page_visibility",
]
},
{
type: "category",
label: "UI Usage Tracking",
@ -292,6 +317,7 @@ const sidebars = {
label: "UI Logs",
items: [
"proxy/ui_logs",
"proxy/ui_spend_log_settings",
"proxy/ui_logs_sessions",
"proxy/deleted_keys_teams"
]
@ -364,6 +390,7 @@ const sidebars = {
label: "Load Balancing, Routing, Fallbacks",
href: "https://docs.litellm.ai/docs/routing-load-balancing",
},
"traffic_mirroring",
{
type: "category",
label: "Logging, Alerting, Metrics",
@ -418,6 +445,7 @@ const sidebars = {
label: "Spend Tracking",
items: [
"proxy/cost_tracking",
"proxy/request_tags",
"proxy/custom_pricing",
"proxy/pricing_calculator",
"proxy/provider_margins",
@ -444,6 +472,7 @@ const sidebars = {
label: "/a2a - A2A Agent Gateway",
items: [
"a2a",
"a2a_invoking_agents",
"a2a_cost_tracking",
"a2a_agent_permissions"
],
@ -513,6 +542,7 @@ const sidebars = {
items: [
"mcp",
"mcp_usage",
"mcp_semantic_filter",
"mcp_control",
"mcp_cost",
"mcp_guardrail",
@ -691,6 +721,7 @@ const sidebars = {
"providers/bedrock_agents",
"providers/bedrock_writer",
"providers/bedrock_batches",
"providers/bedrock_realtime_with_audio",
"providers/aws_polly",
"providers/bedrock_vector_store",
]
@ -775,6 +806,7 @@ const sidebars = {
"providers/oci",
"providers/ollama",
"providers/openrouter",
"providers/sarvam",
"providers/ovhcloud",
"providers/perplexity",
"providers/petals",
@ -822,7 +854,14 @@ const sidebars = {
"providers/watsonx/audio_transcription",
]
},
"providers/xai",
{
type: "category",
label: "xAI",
items: [
"providers/xai",
"providers/xai_realtime",
]
},
"providers/xiaomi_mimo",
"providers/xinference",
"providers/zai",
@ -921,7 +960,6 @@ const sidebars = {
type: "category",
label: "LiteLLM Python SDK Tutorials",
items: [
'tutorials/google_adk',
'tutorials/azure_openai',
'tutorials/instructor',
"tutorials/gradio_integration",
@ -1017,6 +1055,7 @@ const sidebars = {
type: "category",
label: "Issue Reporting",
items: [
"troubleshoot/prisma_migrations",
"troubleshoot/cpu_issues",
"troubleshoot/memory_issues",
"troubleshoot/spend_queue_warnings",

123
docs/my-website/src/theme/BlogListPage/index.js Normal file
View file

@ -0,0 +1,123 @@
import React from 'react';
import Layout from '@theme/Layout';
import Link from '@docusaurus/Link';
import styles from './styles.module.css';
const TAG_COLORS = {
gemini: {bg: '#d2e3fc', text: '#174ea6', darkBg: '#1a3a5c', darkText: '#8ab4f8'},
anthropic: {bg: '#fde0c4', text: '#b33d00', darkBg: '#4a2800', darkText: '#ffb74d'},
claude: {bg: '#fde0c4', text: '#b33d00', darkBg: '#4a2800', darkText: '#ffb74d'},
llms: {bg: '#c8e6c9', text: '#1b5e20', darkBg: '#1b3d1f', darkText: '#81c784'},
};
function hashHue(str) {
let hash = 0;
for (let i = 0; i < str.length; i++) {
hash = str.charCodeAt(i) + ((hash << 5) - hash);
}
return Math.abs(hash) % 360;
}
function getTagColor(label) {
const key = label.toLowerCase();
for (const [k, v] of Object.entries(TAG_COLORS)) {
if (key === k) return v;
}
const hue = hashHue(key);
return {
bg: `hsl(${hue}, 40%, 90%)`,
text: `hsl(${hue}, 60%, 25%)`,
darkBg: `hsl(${hue}, 40%, 20%)`,
darkText: `hsl(${hue}, 50%, 75%)`,
};
}
function formatDate(dateStr) {
const d = new Date(dateStr);
const now = new Date();
const diffDays = Math.floor((now - d) / (1000 * 60 * 60 * 24));
if (diffDays <= 0) return 'Today';
if (diffDays === 1) return '1d ago';
if (diffDays < 30) return `${diffDays}d ago`;
return d.toLocaleDateString('en-US', {month: 'short', day: 'numeric', year: 'numeric'});
}
function BlogCard({post, featured}) {
const {title, permalink, date, description, tags} = post;
const visibleTags = (tags || []).slice(0, 3);
return (
<Link to={permalink} className={styles.cardLink} aria-label={title}>
<article className={featured ? styles.cardFeatured : styles.card}>
<div className={styles.meta}>
<time className={styles.time} dateTime={date}>{formatDate(date)}</time>
{featured && <span className={styles.badge}>Latest</span>}
</div>
<h2 className={styles.title}>{title}</h2>
{description && <p className={styles.desc}>{description}</p>}
{visibleTags.length > 0 && (
<div className={styles.tags}>
{visibleTags.map(tag => {
const c = getTagColor(tag.label);
return (
<span key={tag.label} className={styles.tag} style={{
'--tag-bg': c.bg, '--tag-text': c.text,
'--tag-bg-dark': c.darkBg, '--tag-text-dark': c.darkText,
}}>{tag.label}</span>
);
})}
</div>
)}
<div className={styles.arrow} aria-hidden="true">
<svg width="16" height="16" viewBox="0 0 16 16" fill="none">
<path d="M6 3l5 5-5 5" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round"/>
</svg>
</div>
</article>
</Link>
);
}
function Pagination({metadata}) {
const {previousPage, nextPage} = metadata;
if (!previousPage && !nextPage) return null;
return (
<nav className={styles.pagination} aria-label="Blog list pagination">
{previousPage ? (
<Link to={previousPage} className={styles.paginationLink}>&larr; Newer posts</Link>
) : <span />}
{nextPage ? (
<Link to={nextPage} className={styles.paginationLink}>Older posts &rarr;</Link>
) : <span />}
</nav>
);
}
export default function BlogListPage(props) {
const items = props.items || [];
const metadata = props.metadata || {};
const [first, ...rest] = items;
return (
<Layout
title={metadata.blogTitle || 'Blog'}
description={metadata.blogDescription || 'Guides, announcements, and best practices from the LiteLLM team.'}
>
<header className={styles.hero}>
<h1 className={styles.heroTitle}>The LiteLLM Blog</h1>
<p className={styles.heroSubtitle}>Guides, announcements, and best practices from the LiteLLM team.</p>
</header>
<main className={styles.grid}>
{first && (
<BlogCard post={first.content.metadata} featured />
)}
{rest.map(({content}) => (
<BlogCard key={content.metadata.permalink} post={content.metadata} />
))}
</main>
<Pagination metadata={metadata} />
</Layout>
);
}

163
docs/my-website/src/theme/BlogListPage/styles.module.css Normal file
View file

@ -0,0 +1,163 @@
.hero {
max-width: 960px;
margin: 0 auto;
padding: 3rem 1.5rem 1rem;
text-align: center;
}
.heroTitle {
font-size: 2.25rem;
font-weight: 700;
margin-bottom: 0.25rem;
letter-spacing: -0.02em;
}
.heroSubtitle {
color: var(--ifm-color-emphasis-600);
font-size: 1.1rem;
margin-bottom: 0;
}
.grid {
max-width: 960px;
margin: 0 auto;
padding: 1.5rem;
display: grid;
gap: 1rem;
}
.cardLink {
display: block;
text-decoration: none;
color: inherit;
}
.card {
position: relative;
border: 1px solid var(--ifm-color-emphasis-200);
border-radius: 12px;
padding: 1.5rem;
padding-right: 2.5rem;
height: 100%;
transition: border-color 0.15s, transform 0.15s, background 0.15s;
background: var(--ifm-background-surface-color, var(--ifm-background-color));
}
.card:hover {
border-color: var(--ifm-color-primary);
transform: translateY(-2px);
background: var(--ifm-color-emphasis-100);
}
.cardFeatured {
composes: card;
border-color: var(--ifm-color-primary-lighter);
background: var(--ifm-color-emphasis-100);
}
.meta {
display: flex;
align-items: center;
gap: 0.5rem;
margin-bottom: 0.5rem;
}
.time {
font-size: 0.8rem;
font-weight: 500;
color: var(--ifm-color-emphasis-600);
text-transform: uppercase;
letter-spacing: 0.04em;
}
.badge {
font-size: 0.65rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.06em;
padding: 2px 8px;
border-radius: 99px;
background: var(--ifm-color-primary);
color: #fff;
}
.title {
font-size: 1.15rem;
font-weight: 600;
margin: 0 0 0.4rem;
line-height: 1.35;
}
.desc {
font-size: 0.88rem;
color: var(--ifm-color-emphasis-700);
line-height: 1.5;
margin: 0 0 0.75rem;
}
.tags {
display: flex;
gap: 6px;
flex-wrap: wrap;
}
.tag {
font-size: 0.7rem;
font-weight: 500;
padding: 2px 10px;
border-radius: 99px;
background: var(--tag-bg);
color: var(--tag-text);
}
:global([data-theme='dark']) .tag {
background: var(--tag-bg-dark);
color: var(--tag-text-dark);
}
.arrow {
position: absolute;
right: 1rem;
top: 50%;
transform: translateY(-50%);
color: var(--ifm-color-emphasis-400);
transition: color 0.15s, transform 0.15s;
}
.card:hover .arrow {
color: var(--ifm-color-primary);
transform: translateY(-50%) translateX(3px);
}
.pagination {
max-width: 960px;
margin: 0 auto;
padding: 1rem 1.5rem 3rem;
display: flex;
justify-content: space-between;
}
.paginationLink {
font-size: 0.9rem;
font-weight: 500;
color: var(--ifm-color-primary);
text-decoration: none;
}
.paginationLink:hover {
text-decoration: underline;
}
@media (min-width: 640px) {
.grid {
grid-template-columns: repeat(2, 1fr);
}
.grid .cardLink:first-child {
grid-column: 1 / -1;
}
.grid .cardLink:last-child:nth-child(even) {
grid-column: 1 / -1;
}
}

BIN
enterprise/dist/litellm_enterprise-0.1.29-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.29.tar.gz vendored Normal file
View file

Binary file not shown.

2
enterprise/litellm_enterprise/proxy/auth/route_checks.py
View file

@ -36,7 +36,7 @@ class EnterpriseRouteChecks:
if not premium_user:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"🚨🚨🚨 DISABLING LLM API ENDPOINTS is an Enterprise feature\n🚨 {CommonProxyErrors.not_premium_user.value}",
detail=f"🚨🚨🚨 DISABLING ADMIN ENDPOINTS is an Enterprise feature\n🚨 {CommonProxyErrors.not_premium_user.value}",
)
return get_secret_bool("DISABLE_ADMIN_ENDPOINTS") is True

2
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -53,7 +53,7 @@ class CheckBatchCost:
jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
where={
"status": "validating",
"status": {"in": ["validating", "in_progress", "finalizing"]},
"file_purpose": "batch",
}
)

180
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -166,7 +166,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"updated_by": user_api_key_dict.user_id,
"status": file_object.status,
},
"update": {}, # don't do anything if it already exists
"update": {
"file_object": file_object.model_dump_json(),
"status": file_object.status,
"updated_by": user_api_key_dict.user_id,
}, # FIX: Update status and file_object on every operation to keep state in sync
},
)
@ -244,6 +248,78 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return managed_object.created_by == user_id
return True # don't raise error if managed object is not found
async def list_user_batches(
self,
user_api_key_dict: UserAPIKeyAuth,
limit: Optional[int] = None,
after: Optional[str] = None,
provider: Optional[str] = None,
target_model_names: Optional[str] = None,
llm_router: Optional[Router] = None,
) -> Dict[str, Any]:
# Provider filtering is not supported for managed batches
# This is because the encoded object ids stored in the managed objects table do not contain the provider information
# To support provider filtering, we would need to store the provider information in the encoded object ids
if provider:
raise Exception(
"Filtering by 'provider' is not supported when using managed batches."
)
# Model name filtering is not supported for managed batches
# This is because the encoded object ids stored in the managed objects table do not contain the model name
# A hash of the model name + litellm_params for the model name is encoded as the model id. This is not sufficient to reliably map the target model names to the model ids.
if target_model_names:
raise Exception(
"Filtering by 'target_model_names' is not supported when using managed batches."
)
where_clause: Dict[str, Any] = {"file_purpose": "batch"}
# Filter by user who created the batch
if user_api_key_dict.user_id:
where_clause["created_by"] = user_api_key_dict.user_id
if after:
where_clause["id"] = {"gt": after}
# Fetch more than needed to allow for post-fetch filtering
fetch_limit = limit or 20
if target_model_names:
# Fetch extra to account for filtering
fetch_limit = max(fetch_limit * 3, 100)
batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
where=where_clause,
take=fetch_limit,
order={"created_at": "desc"},
)
batch_objects: List[LiteLLMBatch] = []
for batch in batches:
try:
# Stop once we have enough after filtering
if len(batch_objects) >= (limit or 20):
break
batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
batch_obj = LiteLLMBatch(**batch_data)
batch_obj.id = batch.unified_object_id
batch_objects.append(batch_obj)
except Exception as e:
verbose_logger.warning(
f"Failed to parse batch object {batch.unified_object_id}: {e}"
)
continue
return {
"object": "list",
"data": batch_objects,
"first_id": batch_objects[0].id if batch_objects else None,
"last_id": batch_objects[-1].id if batch_objects else None,
"has_more": len(batch_objects) == (limit or 20),
}
async def get_user_created_file_ids(
self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
) -> List[OpenAIFileObject]:
@ -282,6 +358,31 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
return False
async def check_file_ids_access(
self, file_ids: List[str], user_api_key_dict: UserAPIKeyAuth
) -> None:
"""
Check if the user has access to a list of file IDs.
Only checks managed (unified) file IDs.
Args:
file_ids: List of file IDs to check access for
user_api_key_dict: User API key authentication details
Raises:
HTTPException: If user doesn't have access to any of the files
"""
for file_id in file_ids:
is_unified_file_id = _is_base64_encoded_unified_file_id(file_id)
if is_unified_file_id:
if not await self.can_user_call_unified_file_id(
file_id, user_api_key_dict
):
raise HTTPException(
status_code=403,
detail=f"User {user_api_key_dict.user_id} does not have access to the file {file_id}",
)
async def async_pre_call_hook( # noqa: PLR0915
self,
user_api_key_dict: UserAPIKeyAuth,
@ -297,6 +398,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if (
call_type == CallTypes.afile_content.value
or call_type == CallTypes.afile_delete.value
or call_type == CallTypes.afile_retrieve.value
or call_type == CallTypes.afile_content.value
):
await self.check_managed_file_id_access(data, user_api_key_dict)
@ -313,6 +416,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if messages:
file_ids = self.get_file_ids_from_messages(messages)
if file_ids:
# Check user has access to all managed files
await self.check_file_ids_access(file_ids, user_api_key_dict)
# Check if any files are stored in storage backends and need base64 conversion
# This is needed for Vertex AI/Gemini which requires base64 content
is_vertex_ai = model and ("vertex_ai" in model or "gemini" in model.lower())
@ -328,15 +434,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
data["model_file_id_mapping"] = model_file_id_mapping
elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value:
# Handle managed files in responses API input
# Handle managed files in responses API input and tools
file_ids = []
# Extract file IDs from input parameter
input_data = data.get("input")
if input_data:
file_ids = self.get_file_ids_from_responses_input(input_data)
if file_ids:
model_file_id_mapping = await self.get_model_file_id_mapping(
file_ids, user_api_key_dict.parent_otel_span
)
data["model_file_id_mapping"] = model_file_id_mapping
file_ids.extend(self.get_file_ids_from_responses_input(input_data))
# Extract file IDs from tools parameter (e.g., code_interpreter container)
tools = data.get("tools")
if tools:
file_ids.extend(self.get_file_ids_from_responses_tools(tools))
if file_ids:
# Check user has access to all managed files
await self.check_file_ids_access(file_ids, user_api_key_dict)
model_file_id_mapping = await self.get_model_file_id_mapping(
file_ids, user_api_key_dict.parent_otel_span
)
data["model_file_id_mapping"] = model_file_id_mapping
elif call_type == CallTypes.afile_content.value:
retrieve_file_id = cast(Optional[str], data.get("file_id"))
potential_file_id = (
@ -361,12 +479,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
data["model_file_id_mapping"] = model_file_id_mapping
elif (
call_type == CallTypes.aretrieve_batch.value
or call_type == CallTypes.acancel_batch.value
or call_type == CallTypes.acancel_fine_tuning_job.value
or call_type == CallTypes.aretrieve_fine_tuning_job.value
):
accessor_key: Optional[str] = None
retrieve_object_id: Optional[str] = None
if call_type == CallTypes.aretrieve_batch.value:
if (
call_type == CallTypes.aretrieve_batch.value
or call_type == CallTypes.acancel_batch.value
):
accessor_key = "batch_id"
elif (
call_type == CallTypes.acancel_fine_tuning_job.value
@ -534,6 +656,41 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return file_ids
def get_file_ids_from_responses_tools(
self, tools: List[Dict[str, Any]]
) -> List[str]:
"""
Gets file ids from responses API tools parameter.
The tools can contain code_interpreter with container.file_ids:
[
{
"type": "code_interpreter",
"container": {"type": "auto", "file_ids": ["file-123", "file-456"]}
}
]
"""
file_ids: List[str] = []
if not isinstance(tools, list):
return file_ids
for tool in tools:
if not isinstance(tool, dict):
continue
# Check for code_interpreter with container file_ids
if tool.get("type") == "code_interpreter":
container = tool.get("container")
if isinstance(container, dict):
container_file_ids = container.get("file_ids")
if isinstance(container_file_ids, list):
for file_id in container_file_ids:
if isinstance(file_id, str):
file_ids.append(file_id)
return file_ids
async def get_model_file_id_mapping(
self, file_ids: List[str], litellm_parent_otel_span: Span
) -> dict:
@ -673,6 +830,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
bytes=file_objects[0].bytes,
filename=file_objects[0].filename,
status="uploaded",
expires_at=file_objects[0].expires_at,
)
return response
@ -893,8 +1051,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
delete_response = None
specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
if specific_model_file_id_mapping:
# Remove conflicting keys from data to avoid duplicate keyword arguments
filtered_data = {k: v for k, v in data.items() if k not in ("model", "file_id")}
for model_id, model_file_id in specific_model_file_id_mapping.items():
delete_response = await llm_router.afile_delete(model=model_id, file_id=model_file_id, **data) # type: ignore
delete_response = await llm_router.afile_delete(model=model_id, file_id=model_file_id, **filtered_data) # type: ignore
stored_file_object = await self.delete_unified_file_id(
file_id, litellm_parent_otel_span

2
enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py
View file

@ -282,6 +282,8 @@ async def get_vector_store_info(
updated_at=vector_store.get("updated_at") or None,
litellm_credential_name=vector_store.get("litellm_credential_name"),
litellm_params=vector_store.get("litellm_params") or None,
team_id=vector_store.get("team_id"),
user_id=vector_store.get("user_id"),
)
return {"vector_store": vector_store_pydantic_obj}

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.28"
version = "0.1.29"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.28"
version = "0.1.29"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.28-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.28.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.29-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.29.tar.gz vendored Normal file
View file

Binary file not shown.

10
litellm-proxy-extras/litellm_proxy_extras/migrations/20251119131227_add_prompt_versioning/migration.sql
View file

@ -1,12 +1,12 @@
-- DropIndex
DROP INDEX "LiteLLM_PromptTable_prompt_id_key";
DROP INDEX IF EXISTS "LiteLLM_PromptTable_prompt_id_key";
-- AlterTable
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "version" INTEGER NOT NULL DEFAULT 1;
ALTER TABLE "LiteLLM_PromptTable"
ADD COLUMN "version" INTEGER NOT NULL DEFAULT 1;
-- CreateIndex
CREATE INDEX "LiteLLM_PromptTable_prompt_id_idx" ON "LiteLLM_PromptTable"("prompt_id");
CREATE INDEX "LiteLLM_PromptTable_prompt_id_idx" ON "LiteLLM_PromptTable" ("prompt_id");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_key" ON "LiteLLM_PromptTable"("prompt_id", "version");
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_key" ON "LiteLLM_PromptTable" ("prompt_id", "version");

10
litellm-proxy-extras/litellm_proxy_extras/migrations/20260131150814_add_team_user_to_vector_stores/migration.sql Normal file
View file

@ -0,0 +1,10 @@
-- AlterTable
ALTER TABLE "LiteLLM_ManagedVectorStoresTable" ADD COLUMN "team_id" TEXT,
ADD COLUMN "user_id" TEXT;
-- CreateIndex
CREATE INDEX "LiteLLM_ManagedVectorStoresTable_team_id_idx" ON "LiteLLM_ManagedVectorStoresTable"("team_id");
-- CreateIndex
CREATE INDEX "LiteLLM_ManagedVectorStoresTable_user_id_idx" ON "LiteLLM_ManagedVectorStoresTable"("user_id");

11
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -5,6 +5,7 @@ datasource client {
generator client {
provider = "prisma-client-py"
binaryTargets = ["native", "debian-openssl-1.1.x", "debian-openssl-3.0.x", "linux-musl", "linux-musl-openssl-3.0.x"]
}
// Budget / Rate Limits for an org
@ -128,6 +129,7 @@ model LiteLLM_TeamTable {
team_member_permissions String[] @default([])
policies String[] @default([])
model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases
allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
litellm_model_table LiteLLM_ModelTable? @relation(fields: [model_id], references: [id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
@ -159,7 +161,8 @@ model LiteLLM_DeletedTeamTable {
team_member_permissions String[] @default([])
policies String[] @default([])
model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases
allow_team_guardrail_config Boolean @default(false)
// Original timestamps from team creation/updates
created_at DateTime? @map("created_at")
updated_at DateTime? @map("updated_at")
@ -760,6 +763,11 @@ model LiteLLM_ManagedVectorStoresTable {
updated_at DateTime @updatedAt
litellm_credential_name String?
litellm_params Json?
team_id String?
user_id String?
@@index([team_id])
@@index([user_id])
}
// Guardrails table for storing guardrail configurations
@ -768,6 +776,7 @@ model LiteLLM_GuardrailsTable {
guardrail_name String @unique
litellm_params Json
guardrail_info Json?
team_id String?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}

109
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -18,14 +18,15 @@ def str_to_bool(value: Optional[str]) -> bool:
return value.lower() in ("true", "1", "t", "y", "yes")
def _get_prisma_env() -> dict:
"""Get environment variables for Prisma, handling offline mode if configured."""
prisma_env = os.environ.copy()
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
# These env vars prevent Prisma from attempting downloads
prisma_env["NPM_CONFIG_PREFER_OFFLINE"] = "true"
prisma_env["NPM_CONFIG_CACHE"] = os.getenv("NPM_CONFIG_CACHE", "/app/.cache/npm")
prisma_env["NPM_CONFIG_CACHE"] = os.getenv(
"NPM_CONFIG_CACHE", "/app/.cache/npm"
)
return prisma_env
@ -34,29 +35,28 @@ def _get_prisma_command() -> str:
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
# Primary location where Prisma Python package installs the CLI
default_cli_path = "/app/.cache/prisma-python/binaries/node_modules/.bin/prisma"
# Check if custom path is provided (for flexibility)
custom_cli_path = os.getenv("PRISMA_CLI_PATH")
if custom_cli_path and os.path.exists(custom_cli_path):
logger.info(f"Using custom Prisma CLI at {custom_cli_path}")
return custom_cli_path
# Check the default location
if os.path.exists(default_cli_path):
logger.info(f"Using cached Prisma CLI at {default_cli_path}")
return default_cli_path
# If not found, log warning and fall back
logger.warning(
f"Prisma CLI not found at {default_cli_path}. "
"Falling back to Python wrapper (may attempt downloads)"
)
# Fall back to the Python wrapper (will work in online mode)
return "prisma"
class ProxyExtrasDBManager:
@staticmethod
def _get_prisma_dir() -> str:
@ -119,7 +119,7 @@ class ProxyExtrasDBManager:
stdout=open(migration_file, "w"),
check=True,
timeout=30,
env=prisma_env
env=prisma_env,
)
# 3. Mark the migration as applied since it represents current state
@ -134,7 +134,7 @@ class ProxyExtrasDBManager:
],
check=True,
timeout=30,
env=prisma_env
env=prisma_env,
)
return True
@ -159,14 +159,20 @@ class ProxyExtrasDBManager:
@staticmethod
def _roll_back_migration(migration_name: str):
"""Mark a specific migration as rolled back"""
# Set up environment for offline mode if configured
# Set up environment for offline mode if configured
prisma_env = _get_prisma_env()
subprocess.run(
[_get_prisma_command(), "migrate", "resolve", "--rolled-back", migration_name],
[
_get_prisma_command(),
"migrate",
"resolve",
"--rolled-back",
migration_name,
],
timeout=60,
check=True,
capture_output=True,
env=prisma_env
env=prisma_env,
)
@staticmethod
@ -178,7 +184,7 @@ class ProxyExtrasDBManager:
timeout=60,
check=True,
capture_output=True,
env=prisma_env
env=prisma_env,
)
@staticmethod
@ -228,6 +234,8 @@ class ProxyExtrasDBManager:
r"duplicate key value violates",
r"relation .* already exists",
r"constraint .* already exists",
r"does not exist",
r"Can't drop database.* because it doesn't exist",
]
for pattern in idempotent_patterns:
@ -248,7 +256,7 @@ class ProxyExtrasDBManager:
if not database_url:
logger.error("DATABASE_URL not set")
return
diff_dir = (
Path(migrations_dir)
/ "migrations"
@ -283,7 +291,7 @@ class ProxyExtrasDBManager:
check=True,
timeout=60,
stdout=f,
env=_get_prisma_env()
env=_get_prisma_env(),
)
except subprocess.CalledProcessError as e:
logger.warning(f"Failed to generate migration diff: {e.stderr}")
@ -313,7 +321,7 @@ class ProxyExtrasDBManager:
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
env=_get_prisma_env(),
)
logger.info(f"prisma db execute stdout: {result.stdout}")
logger.info("✅ Migration diff applied successfully")
@ -331,12 +339,18 @@ class ProxyExtrasDBManager:
try:
logger.info(f"Resolving migration: {migration_name}")
subprocess.run(
[_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
[
_get_prisma_command(),
"migrate",
"resolve",
"--applied",
migration_name,
],
timeout=60,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
env=_get_prisma_env(),
)
logger.debug(f"Resolved migration: {migration_name}")
except subprocess.CalledProcessError as e:
@ -375,7 +389,7 @@ class ProxyExtrasDBManager:
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
env=_get_prisma_env(),
)
logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@ -397,27 +411,42 @@ class ProxyExtrasDBManager:
)
if migration_match:
failed_migration = migration_match.group(1)
logger.info(
f"Found failed migration: {failed_migration}, marking as rolled back"
)
# Mark the failed migration as rolled back
subprocess.run(
[
_get_prisma_command(),
"migrate",
"resolve",
"--rolled-back",
failed_migration,
],
timeout=60,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
)
logger.info(
f"✅ Migration {failed_migration} marked as rolled back... retrying"
)
if ProxyExtrasDBManager._is_idempotent_error(e.stderr):
logger.info(
f"Migration {failed_migration} failed due to idempotent error (e.g., column already exists), resolving as applied"
)
ProxyExtrasDBManager._roll_back_migration(
failed_migration
)
ProxyExtrasDBManager._resolve_specific_migration(
failed_migration
)
logger.info(
f"✅ Migration {failed_migration} resolved."
)
return True
else:
logger.info(
f"Found failed migration: {failed_migration}, marking as rolled back"
)
# Mark the failed migration as rolled back
subprocess.run(
[
_get_prisma_command(),
"migrate",
"resolve",
"--rolled-back",
failed_migration,
],
timeout=60,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env(),
)
logger.info(
f"✅ Migration {failed_migration} marked as rolled back... retrying"
)
elif (
"P3005" in e.stderr
and "database schema is not empty" in e.stderr

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.27"
version = "0.4.29"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.27"
version = "0.4.29"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

Some files were not shown because too many files have changed in this diff Show more