scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

Merge branch 'main' into feature/databricks-function-call-missing-pass-description

Browse source
This commit is contained in:
Krish Dholakia 2025-09-09 22:42:32 -07:00 • committed by GitHub
commit a80cfe5fb5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
526 changed files with 34225 additions and 10413 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

150
.circleci/config.yml
View file

@ -439,6 +439,7 @@ jobs:
paths:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
@ -469,7 +470,55 @@ jobs:
command: |
pwd
ls
python -m pytest tests/local_testing tests/router_unit_tests --cov=litellm --cov-report=xml -vv -k "router" -x -v --junitxml=test-results/junit.xml --durations=5
python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -x -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_router_coverage.xml
mv .coverage litellm_router_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_router_coverage.xml
- litellm_router_coverage
litellm_router_unit_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "respx==0.22.0"
pip install "pytest-cov==5.0.0"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -486,11 +535,9 @@ jobs:
- litellm_router_coverage.xml
- litellm_router_coverage
litellm_security_tests:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
@ -499,32 +546,67 @@ jobs:
name: Show git commit hash
command: |
echo "Git commit hash: $CIRCLE_SHA1"
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
- run:
name: Install Python 3.9
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda activate myenv
python --version
- run:
name: Install Dependencies
command: |
pip install "pytest==7.3.1"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install mypy
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
pip install numpydoc
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "gunicorn==21.2.0"
pip install "anyio==3.7.1"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.100.1"
pip install "pytest-cov==5.0.0"
pip install "apscheduler"
- run:
name: Install Trivy
name: Install dockerize
command: |
sudo apt-get update
sudo apt-get install wget apt-transport-https gnupg lsb-release
wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
sudo apt-get update
sudo apt-get install trivy
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Run Trivy scan on LiteLLM Docs
name: Run Security Scans
command: |
trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
- run:
name: Run Trivy scan on LiteLLM UI
command: |
trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
chmod +x ci_cd/security_scans.sh
./ci_cd/security_scans.sh
- run:
name: Run prisma ./docker/entrypoint.sh
command: |
@ -1242,6 +1324,7 @@ jobs:
pip install jinja2
pip install "tokenizers==0.20.0"
pip install "uvloop==0.21.0"
pip install "fastuuid==0.12.0"
pip install jsonschema
- setup_litellm_enterprise_pip
- run:
@ -1374,6 +1457,7 @@ jobs:
# - run: python ./tests/documentation_tests/test_general_setting_keys.py
- run: python ./tests/code_coverage_tests/check_licenses.py
- run: python ./tests/code_coverage_tests/router_code_coverage.py
- run: python ./tests/code_coverage_tests/info_log_check.py
- run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
- run: python ./tests/code_coverage_tests/code_qa_check_tests.py
- run: python ./tests/code_coverage_tests/test_proxy_types_import.py
@ -1427,6 +1511,7 @@ jobs:
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$PROXY_DATABASE_URL \
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e DISABLE_SCHEMA_UPDATE="True" \
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
@ -1542,23 +1627,6 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Install Grype
command: |
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
- run:
name: Build and Scan Docker Images
command: |
# Build and scan Dockerfile.database
echo "Building and scanning Dockerfile.database..."
docker build -t litellm-database:latest -f ./docker/Dockerfile.database .
grype litellm-database:latest --fail-on critical
# Build and scan main Dockerfile
echo "Building and scanning main Dockerfile..."
docker build -t litellm:latest .
grype litellm:latest --fail-on critical
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@ -1862,6 +1930,7 @@ jobs:
-e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
@ -2912,6 +2981,7 @@ jobs:
command: |
docker run --name my-app \
-p 4000:4000 \
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
myapp:latest \
--port 4000 > docker_output.log 2>&1 || true
@ -2982,6 +3052,12 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_router_unit_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- check_code_and_doc_quality:
filters:
branches:
@ -3128,6 +3204,7 @@ workflows:
- image_gen_testing
- logging_testing
- litellm_router_testing
- litellm_router_unit_testing
- caching_unit_tests
- litellm_proxy_unit_testing
- litellm_security_tests
@ -3187,6 +3264,7 @@ workflows:
- image_gen_testing
- logging_testing
- litellm_router_testing
- litellm_router_unit_testing
- caching_unit_tests
- langfuse_logging_unit_tests
- litellm_assistants_api_testing

3
.circleci/requirements.txt
View file

@ -14,4 +14,5 @@ google-cloud-iam==2.19.1
fastapi-sso==0.16.0
uvloop==0.21.0
mcp==1.10.1 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
semantic_router==0.1.10 # for auto-routing with litellm
fastuuid==0.12.0

60
.github/workflows/auto_update_price_and_context_window_file.py vendored
View file

@ -43,8 +43,8 @@ def write_to_file(file_path, data):
# Print an error message if writing to file fails
print("Error updating JSON file:", e)
# Update the existing models and add the missing models
def transform_remote_data(data):
# Update the existing models and add the missing models for OpenRouter
def transform_openrouter_data(data):
transformed = {}
for row in data:
# Add the fields 'max_tokens' and 'input_cost_per_token'
@ -81,6 +81,34 @@ def transform_remote_data(data):
return transformed
# Update the existing models and add the missing models for Vercel AI Gateway
def transform_vercel_ai_gateway_data(data):
transformed = {}
for row in data:
obj = {
"max_tokens": row["context_window"],
"input_cost_per_token": float(row["pricing"]["input"]),
"output_cost_per_token": float(row["pricing"]["output"]),
'max_output_tokens': row['max_tokens'],
'max_input_tokens': row["context_window"],
}
# Handle cache pricing if available
if "pricing" in row:
if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
mode = "embedding" if "embedding" in row["id"].lower() else "chat"
obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
return transformed
# Load local data from a specified file
def load_local_data(file_path):
@ -100,22 +128,32 @@ def load_local_data(file_path):
def main():
local_file_path = "model_prices_and_context_window.json" # Path to the local data file
url = "https://openrouter.ai/api/v1/models" # URL to fetch remote data
openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data
vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data
# Load local data from file
local_data = load_local_data(local_file_path)
# Fetch remote data asynchronously
remote_data = asyncio.run(fetch_data(url))
# Transform the fetched remote data
remote_data = transform_remote_data(remote_data)
# Fetch OpenRouter data
openrouter_data = asyncio.run(fetch_data(openrouter_url))
# Transform the fetched OpenRouter data
openrouter_data = transform_openrouter_data(openrouter_data)
# Fetch Vercel AI Gateway data
vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
# Transform the fetched Vercel AI Gateway data
vercel_data = transform_vercel_ai_gateway_data(vercel_data)
# Combine both datasets
all_remote_data = {**openrouter_data, **vercel_data}
# If both local and remote data are available, synchronize and save
if local_data and remote_data:
sync_local_data_with_remote(local_data, remote_data)
# If both local and openrouter data are available, synchronize and save
if local_data and all_remote_data:
sync_local_data_with_remote(local_data, all_remote_data)
write_to_file(local_file_path, local_data)
else:
print("Failed to fetch model data from either local file or URL.")
# Entry point of the script
if __name__ == "__main__":
main()
main()

1
.github/workflows/test-litellm.yml vendored
View file

@ -31,6 +31,7 @@ jobs:
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
poetry run pip install "google-genai==1.22.0"
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
- name: Setup litellm-enterprise as local package
run: |

2
Makefile
View file

@ -48,7 +48,7 @@ install-test-deps: install-proxy-dev
cd enterprise && python -m pip install -e . && cd ..
install-helm-unittest:
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
# Formatting
format: install-dev

1
README.md
View file

@ -344,6 +344,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
| [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | ✅ | | |
| [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | ✅ | | |
| [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [Heroku](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | | | | |
[**Read the Docs**](https://docs.litellm.ai/docs/)

105
ci_cd/security_scans.sh Executable file
View file

@ -0,0 +1,105 @@
#!/bin/bash
# Security Scans Script for LiteLLM
# This script runs comprehensive security scans including Trivy and Grype
set -e
echo "Starting security scans for LiteLLM..."
# Function to install Trivy and required tools
install_trivy() {
echo "Installing Trivy and required tools..."
sudo apt-get update
sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
sudo apt-get update
sudo apt-get install trivy
echo "Trivy and required tools installed successfully"
}
# Function to install Grype
install_grype() {
echo "Installing Grype..."
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
echo "Grype installed successfully"
}
# Function to run Trivy scans
run_trivy_scans() {
echo "Running Trivy scans..."
echo "Scanning LiteLLM Docs..."
trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
echo "Scanning LiteLLM UI..."
trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
echo "Trivy scans completed successfully"
}
# Function to build and scan Docker images with Grype
run_grype_scans() {
echo "Running Grype scans..."
# Temporarily add wheel files to .dockerignore for security scans
echo "Temporarily modifying .dockerignore to exclude problematic wheel files..."
cp .dockerignore .dockerignore.backup 2>/dev/null || touch .dockerignore.backup
echo "/*.whl" >> .dockerignore
# Build and scan Dockerfile.database
echo "Building and scanning Dockerfile.database..."
docker build -t litellm-database:latest -f ./docker/Dockerfile.database .
grype litellm-database:latest --fail-on critical
# Build and scan main Dockerfile
echo "Building and scanning main Dockerfile..."
docker build -t litellm:latest .
grype litellm:latest --fail-on critical
# Restore original .dockerignore
echo "Restoring original .dockerignore..."
mv .dockerignore.backup .dockerignore
# Scan the locally built LiteLLM image for vulnerabilities with CVSS >= 4.0
echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
echo "Using locally built image: litellm:latest"
# Run grype scan and check for vulnerabilities with CVSS >= 4.0
echo "Checking for vulnerabilities with CVSS score >= 4.0..."
HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq -r '.matches[] | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) | .vulnerability.id' | wc -l)
if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
echo "ERROR: Found $HIGH_SEVERITY_COUNT vulnerabilities with CVSS score >= 4.0 in litellm:latest"
echo "Detailed vulnerability report:"
grype litellm:latest -o json | jq -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
(.matches[] | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) |
[.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description]) |
@tsv' | column -t -s $'\t'
exit 1
else
echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"
fi
echo "Grype scans completed successfully"
}
# Main execution
main() {
echo "Installing security scanning tools..."
install_trivy
install_grype
echo "Running filesystem vulnerability scans..."
run_trivy_scans
echo "Running Docker image vulnerability scans..."
run_grype_scans
echo "All security scans completed successfully!"
}
# Execute main function
main "$@"

9
ci_cd/security_scans_readme.md Normal file
View file

@ -0,0 +1,9 @@
# Security Scans
## Scans that run:
- Trivy scan on `./docs/` (HIGH/CRITICAL/MEDIUM)
- Trivy scan on `./ui/` (HIGH/CRITICAL/MEDIUM)
- Grype scan on `Dockerfile.database` (fails on CRITICAL)
- Grype scan on main `Dockerfile` (fails on CRITICAL)
- Grype CVSS ≥ 4.0 scan on main `Dockerfile` (fails any vulnerabilities with CVSS ≥ 4.0)

311
cookbook/veo_video_generation.py Normal file
View file

@ -0,0 +1,311 @@
#!/usr/bin/env python3
"""
Complete example for Veo video generation through LiteLLM proxy.
This script demonstrates how to:
1. Generate videos using Google's Veo model
2. Poll for completion status
3. Download the generated video file
Requirements:
- LiteLLM proxy running with Google AI Studio pass-through configured
- Google AI Studio API key with Veo access
"""
import json
import os
import time
import requests
from typing import Optional
class VeoVideoGenerator:
"""Complete Veo video generation client using LiteLLM proxy."""
def __init__(self, base_url: str = "http://localhost:4000/gemini/v1beta",
api_key: str = "sk-1234"):
"""
Initialize the Veo video generator.
Args:
base_url: Base URL for the LiteLLM proxy with Gemini pass-through
api_key: API key for LiteLLM proxy authentication
"""
self.base_url = base_url
self.api_key = api_key
self.headers = {
"x-goog-api-key": api_key,
"Content-Type": "application/json"
}
def generate_video(self, prompt: str) -> Optional[str]:
"""
Initiate video generation with Veo.
Args:
prompt: Text description of the video to generate
Returns:
Operation name if successful, None otherwise
"""
print(f"🎬 Generating video with prompt: '{prompt}'")
url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning"
payload = {
"instances": [{
"prompt": prompt
}]
}
try:
response = requests.post(url, headers=self.headers, json=payload)
response.raise_for_status()
data = response.json()
operation_name = data.get("name")
if operation_name:
print(f"✅ Video generation started: {operation_name}")
return operation_name
else:
print("❌ No operation name returned")
print(f"Response: {json.dumps(data, indent=2)}")
return None
except requests.RequestException as e:
print(f"❌ Failed to start video generation: {e}")
if hasattr(e, 'response') and e.response is not None:
try:
error_data = e.response.json()
print(f"Error details: {json.dumps(error_data, indent=2)}")
except:
print(f"Error response: {e.response.text}")
return None
def wait_for_completion(self, operation_name: str, max_wait_time: int = 600) -> Optional[str]:
"""
Poll operation status until video generation is complete.
Args:
operation_name: Name of the operation to monitor
max_wait_time: Maximum time to wait in seconds (default: 10 minutes)
Returns:
Video URI if successful, None otherwise
"""
print("⏳ Waiting for video generation to complete...")
operation_url = f"{self.base_url}/{operation_name}"
start_time = time.time()
poll_interval = 10 # Start with 10 seconds
while time.time() - start_time < max_wait_time:
try:
print(f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)")
response = requests.get(operation_url, headers=self.headers)
response.raise_for_status()
data = response.json()
# Check for errors
if "error" in data:
print("❌ Error in video generation:")
print(json.dumps(data["error"], indent=2))
return None
# Check if operation is complete
is_done = data.get("done", False)
if is_done:
print("🎉 Video generation complete!")
try:
# Extract video URI from nested response
video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
print(f"📹 Video URI: {video_uri}")
return video_uri
except KeyError as e:
print(f"❌ Could not extract video URI: {e}")
print("Full response:")
print(json.dumps(data, indent=2))
return None
# Wait before next poll, with exponential backoff
time.sleep(poll_interval)
poll_interval = min(poll_interval * 1.2, 30) # Cap at 30 seconds
except requests.RequestException as e:
print(f"❌ Error polling operation status: {e}")
time.sleep(poll_interval)
print(f"⏰ Timeout after {max_wait_time} seconds")
return None
def download_video(self, video_uri: str, output_filename: str = "generated_video.mp4") -> bool:
"""
Download the generated video file.
Args:
video_uri: URI of the video to download (from Google's response)
output_filename: Local filename to save the video
Returns:
True if download successful, False otherwise
"""
print(f"⬇️ Downloading video...")
print(f"Original URI: {video_uri}")
# Convert Google URI to LiteLLM proxy URI
# Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media
if video_uri.startswith("files/"):
download_path = f"{video_uri}:download?alt=media"
else:
download_path = video_uri
litellm_download_url = f"{self.base_url}/{download_path}"
print(f"Download URL: {litellm_download_url}")
try:
# Download with streaming and redirect handling
response = requests.get(
litellm_download_url,
headers=self.headers,
stream=True,
allow_redirects=True # Handle redirects automatically
)
response.raise_for_status()
# Save video file
with open(output_filename, 'wb') as f:
downloaded_size = 0
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
downloaded_size += len(chunk)
# Progress indicator for large files
if downloaded_size % (1024 * 1024) == 0: # Every MB
print(f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB...")
# Verify file was created and has content
if os.path.exists(output_filename):
file_size = os.path.getsize(output_filename)
if file_size > 0:
print(f"✅ Video downloaded successfully!")
print(f"📁 Saved as: {output_filename}")
print(f"📏 File size: {file_size / (1024*1024):.2f} MB")
return True
else:
print("❌ Downloaded file is empty")
os.remove(output_filename)
return False
else:
print("❌ File was not created")
return False
except requests.RequestException as e:
print(f"❌ Download failed: {e}")
if hasattr(e, 'response') and e.response is not None:
print(f"Status code: {e.response.status_code}")
print(f"Response headers: {dict(e.response.headers)}")
return False
def generate_and_download(self, prompt: str, output_filename: str = None) -> bool:
"""
Complete workflow: generate video and download it.
Args:
prompt: Text description for video generation
output_filename: Output filename (auto-generated if None)
Returns:
True if successful, False otherwise
"""
# Auto-generate filename if not provided
if output_filename is None:
timestamp = int(time.time())
safe_prompt = "".join(c for c in prompt[:30] if c.isalnum() or c in (' ', '-', '_')).rstrip()
output_filename = f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
print("=" * 60)
print("🎬 VEO VIDEO GENERATION WORKFLOW")
print("=" * 60)
# Step 1: Generate video
operation_name = self.generate_video(prompt)
if not operation_name:
return False
# Step 2: Wait for completion
video_uri = self.wait_for_completion(operation_name)
if not video_uri:
return False
# Step 3: Download video
success = self.download_video(video_uri, output_filename)
if success:
print("=" * 60)
print("🎉 SUCCESS! Video generation complete!")
print(f"📁 Video saved as: {output_filename}")
print("=" * 60)
else:
print("=" * 60)
print("❌ FAILED! Video generation or download failed")
print("=" * 60)
return success
def main():
"""
Example usage of the VeoVideoGenerator.
Configure these environment variables:
- LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
- LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
"""
# Configuration from environment or defaults
base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
print("🚀 Starting Veo Video Generation Example")
print(f"📡 Using LiteLLM proxy at: {base_url}")
# Initialize generator
generator = VeoVideoGenerator(base_url=base_url, api_key=api_key)
# Example prompts - try different ones!
example_prompts = [
"A cat playing with a ball of yarn in a sunny garden",
"Ocean waves crashing against rocky cliffs at sunset",
"A bustling city street with people walking and cars passing by",
"A peaceful forest with sunlight filtering through the trees"
]
# Use first example or get from user
prompt = example_prompts[0]
print(f"🎬 Using prompt: '{prompt}'")
# Generate and download video
success = generator.generate_and_download(prompt)
if success:
print("\n✅ Example completed successfully!")
print("💡 Try modifying the prompt in the script for different videos!")
else:
print("\n❌ Example failed!")
print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key")
# Troubleshooting tips
print("\n🔍 Troubleshooting:")
print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through")
print("2. Verify your Google AI Studio API key has Veo access")
print("3. Check that your prompt meets Veo's content guidelines")
print("4. Review the LiteLLM proxy logs for detailed error information")
if __name__ == "__main__":
main()

2
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.5
version: 0.4.6
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

43
deploy/charts/litellm-helm/README.md
View file

@ -36,11 +36,50 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` |
| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` |
| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A |
| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | N/A |
| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | `[]` |
| `proxyConfigMap.create` | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it. | `true` |
| `proxyConfigMap.name` | When `create=false`, name of the existing ConfigMap to mount. | `""` |
| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` |
| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.
| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` |
| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` |
| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` |
#### Example `proxy_config` ConfigMap from values (default):
```
proxyConfigMap:
create: true
key: "config.yaml"
proxy_config:
general_settings:
master_key: os.environ/PROXY_MASTER_KEY
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
api_key: eXaMpLeOnLy
```
#### Example using existing `proxyConfigMap` instead of creating it:
```
proxyConfigMap:
create: false
name: my-litellm-config
key: config.yaml
# proxy_config is ignored in this mode
```
#### Example `environmentSecrets` Secret
```
apiVersion: v1
kind: Secret

1
deploy/charts/litellm-helm/templates/NOTES.txt
View file

@ -20,3 +20,4 @@
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.*

4
deploy/charts/litellm-helm/templates/configmap-litellm.yaml
View file

@ -1,7 +1,9 @@
{{- if .Values.proxyConfigMap.create }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "litellm.fullname" . }}-config
data:
config.yaml: |
{{ .Values.proxy_config | toYaml | indent 6 }}
{{ .Values.proxy_config | toYaml | indent 6 }}
{{- end }}

8
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -16,7 +16,9 @@ spec:
template:
metadata:
annotations:
{{- if .Values.proxyConfigMap.create }}
checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
@ -183,9 +185,13 @@ spec:
{{- end }}
- name: litellm-config
configMap:
{{- if .Values.proxyConfigMap.create }}
name: {{ include "litellm.fullname" . }}-config
{{- else }}
name: {{ .Values.proxyConfigMap.name }}
{{- end }}
items:
- key: "config.yaml"
- key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
path: "config.yaml"
{{- with .Values.volumes }}
{{- toYaml . | nindent 8 }}

2
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -61,7 +61,7 @@ spec:
value: {{ .Values.db.database }}
- name: DATABASE_URL
value: {{ .Values.db.url | quote }}
{{- else }}
{{- else if .Values.db.deployStandalone }}
- name: DATABASE_URL
value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }}
{{- end }}

33
deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml Normal file
View file

@ -0,0 +1,33 @@
{{- /*
PodDisruptionBudget for LiteLLM proxy
Controlled via .Values.pdb.enabled and .Values.pdb.{minAvailable|maxUnavailable}
Only one of minAvailable / maxUnavailable should be set. If both are set, minAvailable wins.
*/ -}}
{{- if .Values.pdb.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- with .Values.pdb.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.pdb.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
{{- /* Match the Deployment selector to target the same pod set */ -}}
{{- include "litellm.selectorLabels" . | nindent 6 }}
{{- if .Values.pdb.minAvailable }}
minAvailable: {{ .Values.pdb.minAvailable }}
{{- else if .Values.pdb.maxUnavailable }}
maxUnavailable: {{ .Values.pdb.maxUnavailable }}
{{- else }}
# Safe default if enabled but not configured
maxUnavailable: 1
{{- end }}
{{- end }}

22
deploy/charts/litellm-helm/tests/deployment_tests.yaml
View file

@ -115,3 +115,25 @@ tests:
content:
name: EXTRA_ENV_VAR
value: EXTRA_ENV_VAR_VALUE
- it: should mount existing configmap when create=false
template: deployment.yaml
set:
proxyConfigMap:
create: false
name: my-litellm-config
key: custom.yaml
asserts:
- contains:
path: spec.template.spec.volumes
content:
name: litellm-config
configMap:
name: my-litellm-config
items:
- key: custom.yaml
path: config.yaml
- contains:
path: spec.template.spec.containers[0].volumeMounts
content:
name: litellm-config
mountPath: /etc/litellm/

16
deploy/charts/litellm-helm/tests/migrations-job_tests.yaml
View file

@ -110,4 +110,18 @@ tests:
path: spec.template.spec.containers[0].env
content:
name: CUSTOM_VAR
value: "custom_value"
value: "custom_value"
- it: should not include DATABASE_URL when deployStandalone is false
template: migrations-job.yaml
set:
migrationJob:
enabled: true
db:
deployStandalone: false
useExisting: false
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL

45
deploy/charts/litellm-helm/tests/pdb_tests.yaml Normal file
View file

@ -0,0 +1,45 @@
suite: "pdb enabled"
templates:
- poddisruptionbudget.yaml
tests:
- it: "renders a PDB with maxUnavailable=1"
set:
pdb.enabled: true
pdb.maxUnavailable: 1
asserts:
- hasDocuments: { count: 1 }
- isKind: { of: PodDisruptionBudget }
- equal: { path: apiVersion, value: policy/v1 }
- equal: { path: spec.maxUnavailable, value: 1 }
- equal:
path: spec.selector.matchLabels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
---
suite: "pdb disabled"
templates:
- poddisruptionbudget.yaml
tests:
- it: "does not render when disabled"
set:
pdb.enabled: false
asserts:
- hasDocuments: { count: 0 }
---
suite: "pdb minAvailable precedence"
templates:
- poddisruptionbudget.yaml
tests:
- it: "uses minAvailable when both are set"
set:
pdb.enabled: true
pdb.minAvailable: "50%"
pdb.maxUnavailable: 1
asserts:
- isKind: { of: PodDisruptionBudget }
- equal: { path: apiVersion, value: policy/v1 }
- equal: { path: spec.minAvailable, value: "50%" }
- isNull: { path: spec.maxUnavailable }

17
deploy/charts/litellm-helm/values.yaml
View file

@ -93,6 +93,14 @@ masterkeySecretName: ""
# if set, use this secret key for the master key; otherwise, use the default key
masterkeySecretKey: ""
proxyConfigMap:
# when true, creates a new configmap
create: true
# if create is false and name is set, use existing ConfigMap
# create: false
# name: ""
# key: "config.yaml"
# The elements within proxy_config are rendered as config.yaml for the proxy
# Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
# Reference: https://docs.litellm.ai/docs/proxy/configs
@ -232,4 +240,11 @@ extraEnvVars: {
# value: EXTRA_ENV_VAR_VALUE
}
# Pod Disruption Budget
pdb:
enabled: false
# Set exactly one of the following. If both are set, minAvailable takes precedence.
minAvailable: null # e.g. "50%" or 1
maxUnavailable: null # e.g. 1 or "20%"
annotations: {}
labels: {}

BIN
dist/litellm-1.57.6.tar.gz vendored
View file

Binary file not shown.

7
docker/Dockerfile.non_root
View file

@ -33,11 +33,12 @@ WORKDIR /app
# Install runtime dependencies
USER root
RUN apk upgrade --no-cache && \
apk add --no-cache bash libstdc++ ca-certificates openssl
apk add --no-cache bash libstdc++ ca-certificates openssl supervisor
# Copy only necessary artifacts from builder stage for runtime
COPY . .
COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
COPY --from=builder /app/schema.prisma /app/schema.prisma
COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
@ -70,7 +71,9 @@ RUN mkdir -p /nonexistent /.npm && \
chown -R nobody:nogroup /app && \
chown -R nobody:nogroup /nonexistent /.npm && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH
chown -R nobody:nogroup $PRISMA_PATH && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
# --- OpenShift Compatibility: Apply Red Hat recommended pattern ---
# Get paths for directories that need write access at runtime

66
docs/my-website/docs/completion/document_understanding.md
View file

@ -10,6 +10,7 @@ Works for:
- Bedrock Models
- Anthropic API Models
- OpenAI API Models
- Mistral (Only using file ID of already uploaded file, similar to OpenAI file_id input)
## Quick Start
@ -279,6 +280,71 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
</Tabs>
## Mistral Example
Here is a sample payload for using the Mistral model for document understanding:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm.utils import completion
# pdf file_id received from files endpoint
file_id = "fa778e5e-46ec-4562-8418-36623fe25a71"
# model
model = "mistral/mistral-large-latest"
file_content = [
{"type": "text", "text": "What's this file about?"},
{
"type": "file",
"file": {
"file_id": file_id,
}
},
]
response = completion(
model=model,
messages=[{"role": "user", "content": file_content}],
)
assert response is not None
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "mistral/mistral-large-latest",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "What is the content of the file?"
},
{
"type": "file",
"file": {
"file_id": "fa778e5e-46ec-4562-8418-36623fe25a71"
}
}
]
}
]
}
```
</TabItem>
</Tabs>
## Checking if a model supports pdf input
<Tabs>

145
docs/my-website/docs/completion/http_handler_config.md Normal file
View file

@ -0,0 +1,145 @@
# Custom HTTP Handler
Configure custom aiohttp sessions for better performance and control in LiteLLM completions.
## Overview
You can now inject custom `aiohttp.ClientSession` instances into LiteLLM for:
- Custom connection pooling and timeouts
- Corporate proxy and SSL configurations
- Performance optimization
- Request monitoring
## Basic Usage
### Default (No Changes Required)
```python
import litellm
# Works exactly as before
response = await litellm.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello!"}]
)
```
### Custom Session
```python
import aiohttp
import litellm
from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler
# Create optimized session
session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=180),
connector=aiohttp.TCPConnector(limit=300, limit_per_host=75)
)
# Replace global handler
litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
# All completions now use your session
response = await litellm.acompletion(model="gpt-3.5-turbo", messages=[...])
```
## Common Patterns
### FastAPI Integration
```python
from contextlib import asynccontextmanager
from fastapi import FastAPI
import aiohttp
import litellm
@asynccontextmanager
async def lifespan(app: FastAPI):
# Startup
session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=180),
connector=aiohttp.TCPConnector(limit=300)
)
litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(
client_session=session
)
yield
# Shutdown
await session.close()
app = FastAPI(lifespan=lifespan)
@app.post("/chat")
async def chat(messages: list[dict]):
return await litellm.acompletion(model="gpt-3.5-turbo", messages=messages)
```
### Corporate Proxy
```python
import ssl
# Custom SSL context
ssl_context = ssl.create_default_context()
ssl_context.load_cert_chain('cert.pem', 'key.pem')
# Proxy session
session = aiohttp.ClientSession(
connector=aiohttp.TCPConnector(ssl=ssl_context),
trust_env=True # Use environment proxy settings
)
litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
```
### High Performance
```python
# Optimized for high throughput
session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=300),
connector=aiohttp.TCPConnector(
limit=1000, # High connection limit
limit_per_host=200, # Per host limit
ttl_dns_cache=600, # DNS cache
keepalive_timeout=60, # Keep connections alive
enable_cleanup_closed=True
)
)
litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
```
## Constructor Options
```python
BaseLLMAIOHTTPHandler(
client_session=None, # Custom aiohttp.ClientSession
transport=None, # Advanced transport control
connector=None, # Custom aiohttp.BaseConnector
)
```
## Resource Management
- **User sessions**: You manage the lifecycle (call `await session.close()`)
- **Auto-created sessions**: Automatically cleaned up by the handler
- **100% backward compatible**: Existing code works unchanged
## Configuration Tips
### Development
```python
session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=60),
connector=aiohttp.TCPConnector(limit=50)
)
```
### Production
```python
session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=300),
connector=aiohttp.TCPConnector(
limit=1000,
limit_per_host=200,
keepalive_timeout=60
)
)
```

232
docs/my-website/docs/completion/image_generation_chat.md Normal file
View file

@ -0,0 +1,232 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Image Generation in Chat Completions, Responses API
This guide covers how to generate images when using the `chat/completions`. Note - if you want this on Responses API please file a Feature Request [here](https://github.com/BerriAI/litellm/issues/new).
:::info
Requires LiteLLM v1.76.1+
:::
Supported Providers:
- Google AI Studio (`gemini`)
- Vertex AI (`vertex_ai/`)
LiteLLM will standardize the `image` response in the assistant message for models that support image generation during chat completions.
```python title="Example response from litellm"
"message": {
...
"content": "Here's the image you requested:",
"image": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
}
```
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Image generation with chat completion"
from litellm import completion
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = completion(
model="gemini/gemini-2.5-flash-image-preview",
messages=[
{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
],
)
print(response.choices[0].message.content) # Text response
print(response.choices[0].message.image) # Image data
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gemini-image-gen
litellm_params:
model: gemini/gemini-2.5-flash-image-preview
api_key: os.environ/GEMINI_API_KEY
```
2. Run proxy server
```bash showLineNumbers title="Start the proxy"
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
```bash showLineNumbers title="Make request"
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gemini-image-gen",
"messages": [
{
"role": "user",
"content": "Generate an image of a banana wearing a costume that says LiteLLM"
}
]
}'
```
</TabItem>
</Tabs>
**Expected Response**
```bash
{
"id": "chatcmpl-3b66124d79a708e10c603496b363574c",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Here's the image you requested:",
"role": "assistant",
"image": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
}
}
],
"created": 1723323084,
"model": "gemini/gemini-2.5-flash-image-preview",
"object": "chat.completion",
"usage": {
"completion_tokens": 12,
"prompt_tokens": 16,
"total_tokens": 28
}
}
```
## Streaming Support
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Streaming image generation"
from litellm import completion
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = completion(
model="gemini/gemini-2.5-flash-image-preview",
messages=[
{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
],
stream=True,
)
for chunk in response:
if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None:
print("Generated image:", chunk.choices[0].delta.image["url"])
break
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash showLineNumbers title="Streaming request"
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gemini-image-gen",
"messages": [
{
"role": "user",
"content": "Generate an image of a banana wearing a costume that says LiteLLM"
}
],
"stream": true
}'
```
</TabItem>
</Tabs>
**Expected Streaming Response**
```bash
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"content":"Here's the image you requested:"},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"image":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...","detail":"auto"}},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
data: [DONE]
```
## Async Support
```python showLineNumbers title="Async image generation"
from litellm import acompletion
import asyncio
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
async def generate_image():
response = await acompletion(
model="gemini/gemini-2.5-flash-image-preview",
messages=[
{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
],
)
print(response.choices[0].message.content) # Text response
print(response.choices[0].message.image) # Image data
return response
# Run the async function
asyncio.run(generate_image())
```
## Supported Models
| Provider | Model |
|----------|--------|
| Google AI Studio | `gemini/gemini-2.5-flash-image-preview` |
| Vertex AI | `vertex_ai/gemini-2.5-flash-image-preview` |
## Spec
The `image` field in the response follows this structure:
```python
"image": {
"url": "data:image/png;base64,<base64_encoded_image>",
"detail": "auto"
}
```
- `url` - str: Base64 encoded image data in data URI format
- `detail` - str: Image detail level (always "auto" for generated images)
The image is returned as a base64-encoded data URI that can be directly used in HTML `<img>` tags or saved to a file.

3
docs/my-website/docs/completion/input.md
View file

@ -106,6 +106,7 @@ def completion(
parallel_tool_calls: Optional[bool] = None,
logprobs: Optional[bool] = None,
top_logprobs: Optional[int] = None,
safety_identifier: Optional[str] = None,
deployment_id=None,
# soon to be deprecated params by OpenAI
functions: Optional[List] = None,
@ -196,6 +197,8 @@ def completion(
- `top_logprobs`: *int (optional)* - An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to true if this parameter is used.
- `safety_identifier`: *string (optional)* - A unique identifier for tracking and managing safety-related requests. This parameter helps with safety monitoring and compliance tracking.
- `headers`: *dict (optional)* - A dictionary of headers to be sent with the request.
- `extra_headers`: *dict (optional)* - Alternative to `headers`, used to send extra headers in LLM API request.

59
docs/my-website/docs/completion/web_search.md
View file

@ -8,10 +8,25 @@ Use web search with litellm
| Feature | Details |
|---------|---------|
| Supported Endpoints | - `/chat/completions` <br/> - `/responses` |
| Supported Providers | `openai`, `xai`, `vertex_ai`, `gemini`, `perplexity` |
| Supported Providers | `openai`, `xai`, `vertex_ai`, `anthropic`, `gemini`, `perplexity` |
| LiteLLM Cost Tracking | ✅ Supported |
| LiteLLM Version | `v1.71.0+` |
## Which Search Engine is Used?
Each provider uses their own search backend:
| Provider | Search Engine | Notes |
|----------|---------------|-------|
| **OpenAI** (`gpt-4o-search-preview`) | OpenAI's internal search | Real-time web data |
| **xAI** (`grok-3`) | xAI's search + X/Twitter | Real-time social media data |
| **Google AI/Vertex** (`gemini-2.0-flash`) | **Google Search** | Uses actual Google search results |
| **Anthropic** (`claude-3-5-sonnet`) | Anthropic's web search | Real-time web data |
| **Perplexity** | Perplexity's search engine | AI-powered search and reasoning |
:::info
**Anthropic Web Search Models**: Claude models that support web search: `claude-3-5-sonnet-latest`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-latest`, `claude-3-5-haiku-20241022`, `claude-3-7-sonnet-20250219`
:::
## `/chat/completions` (litellm.completion)
@ -56,6 +71,12 @@ model_list:
model: xai/grok-3
api_key: os.environ/XAI_API_KEY
# Anthropic
- model_name: claude-3-5-sonnet-latest
litellm_params:
model: anthropic/claude-3-5-sonnet-latest
api_key: os.environ/ANTHROPIC_API_KEY
# VertexAI
- model_name: gemini-2-flash
litellm_params:
@ -143,6 +164,31 @@ response = completion(
)
```
**Anthropic (using web_search_options)**
```python showLineNumbers
from litellm import completion
# Customize search context size for Anthropic
response = completion(
model="anthropic/claude-3-5-sonnet-latest",
messages=[
{
"role": "user",
"content": "What was a positive news story from today?",
}
],
web_search_options={
"search_context_size": "medium", # Options: "low", "medium" (default), "high"
"user_location": {
"type": "approximate",
"approximate": {
"city": "San Francisco",
},
}
}
)
```
**VertexAI/Gemini (using web_search_options)**
```python showLineNumbers
from litellm import completion
@ -375,6 +421,9 @@ assert litellm.supports_web_search(model="openai/gpt-4o-search-preview") == True
# Check xAI models
assert litellm.supports_web_search(model="xai/grok-3") == True
# Check Anthropic models
assert litellm.supports_web_search(model="anthropic/claude-3-5-sonnet-latest") == True
# Check VertexAI models
assert litellm.supports_web_search(model="gemini-2.0-flash") == True
@ -405,6 +454,14 @@ model_list:
model_info:
supports_web_search: True
# Anthropic
- model_name: claude-3-5-sonnet-latest
litellm_params:
model: anthropic/claude-3-5-sonnet-latest
api_key: os.environ/ANTHROPIC_API_KEY
model_info:
supports_web_search: True
# VertexAI
- model_name: gemini-2-flash
litellm_params:

5
docs/my-website/docs/contributing.md
View file

@ -14,6 +14,11 @@ git clone https://github.com/BerriAI/litellm.git
Tell the proxy where the UI is located
```bash
export PROXY_BASE_URL="http://localhost:3000/"
### ALSO ### - set the basic env variables
DATABASE_URL = "postgresql://<user>:<password>@<host>:<port>/<dbname>"
LITELLM_MASTER_KEY = "sk-1234"
STORE_MODEL_IN_DB = "True"
```
```bash

1
docs/my-website/docs/exception_mapping.md
View file

@ -12,6 +12,7 @@ All exceptions can be imported from `litellm` - e.g. `from litellm import BadReq
| 400 | UnsupportedParamsError | litellm.BadRequestError | Raised when unsupported params are passed |
| 400 | ContextWindowExceededError| litellm.BadRequestError | Special error type for context window exceeded error messages - enables context window fallbacks |
| 400 | ContentPolicyViolationError| litellm.BadRequestError | Special error type for content policy violation error messages - enables content policy fallbacks |
| 400 | ImageFetchError | litellm.BadRequestError | Raised when there are errors fetching or processing images |
| 400 | InvalidRequestError | openai.BadRequestError | Deprecated error, use BadRequestError instead |
| 401 | AuthenticationError | openai.AuthenticationError |
| 403 | PermissionDeniedError | openai.PermissionDeniedError |

220
docs/my-website/docs/extras/gemini_img_migration.md Normal file
View file

@ -0,0 +1,220 @@
# Gemini Image Generation Migration Guide
## Who is impacted by this change?
Anyone using the following models with /chat/completions:
- `gemini/gemini-2.0-flash-exp-image-generation`
- `vertex_ai/gemini-2.0-flash-exp-image-generation`
## Key Change
:::info
From v1.77.0, LiteLLM will return the List of images in `response.choices[0].message.images` instead of a single image in `response.choices[0].message.image`.
:::
Gemini models now support image generation through chat completions. Images are returned in `response.choices[0].message.images` with base64 data URLs.
## Before and After
### Before
```python
from litellm import completion
response = completion(
model="gemini/gemini-2.0-flash-exp-image-generation",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
modalities=["image", "text"],
)
base_64_image_data = response.choices[0].message.content
```
### After
```python
from litellm import completion
response = completion(
model="gemini/gemini-2.0-flash-exp-image-generation",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
modalities=["image", "text"],
)
# Image is now available in the response
image_url = response.choices[0].message.images[0]["image_url"]["url"] # "data:image/png;base64,..."
```
### Why the change?
Because the newer `gemini-2.5-flash-image-preview` model sends both text and image responses in the same response. This interface allows a developer to explicitly access the image or text components of the response. Before a developer would have needed to search through the message content to find the image generated by the model.
**Why the change from `image` to `images`?**
This is to be consistent with the OpenRouter API, making sure we are using simple, well-known interfaces where possible.
## Usage
### Using the Python SDK
**Key Change:**
```diff
# Before
-- base_64_image_data = response.choices[0].message.content
# After
++ image_url = response.choices[0].message.images[0]["image_url"]["url"]
```
#### Basic Image Generation
```python
from litellm import completion
import os
# Set your API key
os.environ["GEMINI_API_KEY"] = "your-api-key"
# Generate an image
response = completion(
model="gemini/gemini-2.0-flash-exp-image-generation",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
modalities=["image", "text"],
)
# Access the generated image
print(response.choices[0].message.content) # Text response (if any)
print(response.choices[0].message.images[0]) # Image data
```
#### Response Format
The image is returned in the `message.images` field:
```python
{
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
},
"index": 0,
"type": "image_url"
}
```
### Using the LiteLLM Proxy Server
**Key Change:**
```diff
# Before
-- "content": "base64-image-data..."
# After
++ "images": [{
++ "image_url": {
++ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
++ "detail": "auto"
++ },
++ "index": 0,
++ "type": "image_url"
++ }]
```
#### Configuration Setup
1. **Configure your models in `config.yaml`:**
```yaml
model_list:
- model_name: gemini-image-gen
litellm_params:
model: gemini/gemini-2.0-flash-exp-image-generation
api_key: os.environ/GEMINI_API_KEY
- model_name: vertex-image-gen
litellm_params:
model: vertex_ai/gemini-2.5-flash-image-preview
vertex_project: your-project-id
vertex_location: us-central1
general_settings:
master_key: sk-1234 # Your proxy API key
```
2. **Start the proxy server:**
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### Making Requests
**Using OpenAI SDK:**
```python
from openai import OpenAI
# Point to your proxy server
client = OpenAI(
api_key="sk-1234", # Your proxy API key
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="gemini-image-gen",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
extra_body={"modalities": ["image", "text"]}
)
# Access the generated image
print(response.choices[0].message.content) # Text response (if any)
print(response.choices[0].message.image) # Image data
```
**Using curl:**
```bash
curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gemini-image-gen",
"messages": [
{
"role": "user",
"content": "Generate an image of a cat"
}
],
"modalities": ["image", "text"]
}'
```
**Response format from proxy:**
```json
{
"id": "chatcmpl-123",
"object": "chat.completion",
"created": 1704089632,
"model": "gemini-image-gen",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "Here's an image of a cat for you!",
"images": [{
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
},
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 8,
"total_tokens": 18
}
}
```

62
docs/my-website/docs/image_edits.md
View file

@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
# /images/edits
LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint.
LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint. Now supports both single and multiple image editing.
| Feature | Supported | Notes |
|---------|-----------|--------|
@ -13,7 +13,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| End-user Tracking | ✅ | |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Supported operations | Create image edits | |
| Supported operations | Create image edits | Single and multiple images supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | |
| Supported LiteLLM Proxy Versions | 1.71.1+ | |
| Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
@ -41,6 +41,26 @@ response = litellm.image_edit(
print(response)
```
#### Multiple Images Edit
```python showLineNumbers title="OpenAI Multiple Images Edit"
import litellm
# Edit multiple images with a prompt
response = litellm.image_edit(
model="gpt-image-1",
image=[
open("image1.png", "rb"),
open("image2.png", "rb"),
open("image3.png", "rb")
],
prompt="Apply vintage filter to all images",
n=1,
size="1024x1024"
)
print(response)
```
#### Image Edit with Mask
```python showLineNumbers title="OpenAI Image Edit with Mask"
import litellm
@ -80,6 +100,30 @@ response = asyncio.run(edit_image())
print(response)
```
#### Async Multiple Images Edit
```python showLineNumbers title="Async OpenAI Multiple Images Edit"
import litellm
import asyncio
async def edit_multiple_images():
response = await litellm.aimage_edit(
model="gpt-image-1",
image=[
open("portrait1.png", "rb"),
open("portrait2.png", "rb")
],
prompt="Add professional lighting to the portraits",
n=1,
size="1024x1024",
response_format="url"
)
return response
# Run the async function
response = asyncio.run(edit_multiple_images())
print(response)
```
#### Image Edit with Custom Parameters
```python showLineNumbers title="OpenAI Image Edit with Custom Parameters"
import litellm
@ -163,6 +207,20 @@ curl -X POST "http://localhost:4000/v1/images/edits" \
-F "response_format=url"
```
#### cURL Multiple Images Example
```bash showLineNumbers title="cURL Multiple Images Edit Request"
curl -X POST "http://localhost:4000/v1/images/edits" \
-H "Authorization: Bearer your-api-key" \
-F "model=gpt-image-1" \
-F "image=@image1.png" \
-F "image=@image2.png" \
-F "image=@image3.png" \
-F "prompt=Apply artistic filter to all images" \
-F "n=1" \
-F "size=1024x1024" \
-F "response_format=url"
```
</TabItem>
</Tabs>

2
docs/my-website/docs/image_generation.md
View file

@ -124,8 +124,6 @@ Any non-openai params, will be treated as provider-specific params, and sent in
- `size`: *string (optional)* The size of the generated images. Must be one of `1024x1024`, `1536x1024` (landscape), `1024x1536` (portrait), or `auto` (default value) for `gpt-image-1`, one of `256x256`, `512x512`, or `1024x1024` for `dall-e-2`, and one of `1024x1024`, `1792x1024`, or `1024x1792` for `dall-e-3`.
- `input_fidelity`: *string (optional)* Controls how closely the model follows the input prompt. Supported for `gpt-image-1` model. Higher fidelity may improve prompt adherence but could affect generation speed.
- `timeout`: *integer* - The maximum time, in seconds, to wait for the API to respond. Defaults to 600 seconds (10 minutes).
- `user`: *string (optional)* A unique identifier representing your end-user,

37
docs/my-website/docs/index.md
View file

@ -226,6 +226,23 @@ response = completion(
</TabItem>
<TabItem value="vercel" label="Vercel AI Gateway">
```python
from litellm import completion
import os
## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
</Tabs>
### Response Format (OpenAI Format)
@ -234,7 +251,7 @@ response = completion(
{
"id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885",
"created": 1734366691,
"model": "claude-3-sonnet-20240229",
"model": "gpt-4o-2024-08-06",
"object": "chat.completion",
"system_fingerprint": null,
"choices": [
@ -446,6 +463,24 @@ response = completion(
</TabItem>
<TabItem value="vercel" label="Vercel AI Gateway">
```python
from litellm import completion
import os
## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
messages = [{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
```
</TabItem>
</Tabs>
### Streaming Response Format (OpenAI Format)

318
docs/my-website/docs/langchain/langchain.md
View file

@ -162,3 +162,321 @@ Get more details [here](../observability/lunary_integration.md)
## Use LangChain ChatLiteLLM + Langfuse
Checkout this section [here](../observability/langfuse_integration#use-langchain-chatlitellm--langfuse) for more details on how to integrate Langfuse with ChatLiteLLM.
## Using Tags with LangChain and LiteLLM
Tags are a powerful feature in LiteLLM that allow you to categorize, filter, and track your LLM requests. When using LangChain with LiteLLM, you can pass tags through the `extra_body` parameter in the metadata.
### Basic Tag Usage
<Tabs>
<TabItem value="openai" label="OpenAI">
```python
import os
from langchain_openai import ChatOpenAI
from langchain_core.messages import HumanMessage, SystemMessage
os.environ['OPENAI_API_KEY'] = "sk-your-key-here"
chat = ChatOpenAI(
model="gpt-4o",
temperature=0.7,
extra_body={
"metadata": {
"tags": ["production", "customer-support", "high-priority"]
}
}
)
messages = [
SystemMessage(content="You are a helpful customer support assistant."),
HumanMessage(content="How do I reset my password?")
]
response = chat.invoke(messages)
print(response)
```
</TabItem>
<TabItem value="anthropic" label="Anthropic">
```python
import os
from langchain_openai import ChatOpenAI
from langchain_core.messages import HumanMessage, SystemMessage
os.environ['ANTHROPIC_API_KEY'] = "sk-ant-your-key-here"
chat = ChatOpenAI(
model="claude-3-sonnet-20240229",
temperature=0.7,
extra_body={
"metadata": {
"tags": ["research", "analysis", "claude-model"]
}
}
)
messages = [
SystemMessage(content="You are a research analyst."),
HumanMessage(content="Analyze this market trend...")
]
response = chat.invoke(messages)
print(response)
```
</TabItem>
<TabItem value="litellm-proxy" label="LiteLLM Proxy">
```python
import os
from langchain_openai import ChatOpenAI
from langchain_core.messages import HumanMessage, SystemMessage
# No API key needed when using proxy
chat = ChatOpenAI(
openai_api_base="http://localhost:4000", # Your proxy URL
model="gpt-4o",
temperature=0.7,
extra_body={
"metadata": {
"tags": ["proxy", "team-alpha", "feature-flagged"],
"generation_name": "customer-onboarding",
"trace_user_id": "user-12345"
}
}
)
messages = [
SystemMessage(content="You are an onboarding assistant."),
HumanMessage(content="Welcome our new customer!")
]
response = chat.invoke(messages)
print(response)
```
</TabItem>
</Tabs>
### Advanced Tag Patterns
#### Dynamic Tags Based on Context
```python
import os
from langchain_openai import ChatOpenAI
from langchain_core.messages import HumanMessage, SystemMessage
def create_chat_with_tags(user_type: str, feature: str):
"""Create a chat instance with dynamic tags based on context"""
# Build tags dynamically
tags = ["langchain-integration"]
if user_type == "premium":
tags.extend(["premium-user", "high-priority"])
elif user_type == "enterprise":
tags.extend(["enterprise", "custom-sla"])
else:
tags.append("standard-user")
# Add feature-specific tags
if feature == "code-review":
tags.extend(["development", "code-analysis"])
elif feature == "content-gen":
tags.extend(["marketing", "content-creation"])
return ChatOpenAI(
openai_api_base="http://localhost:4000",
model="gpt-4o",
temperature=0.7,
extra_body={
"metadata": {
"tags": tags,
"user_type": user_type,
"feature": feature,
"trace_user_id": f"user-{user_type}-{feature}"
}
}
)
# Usage examples
premium_chat = create_chat_with_tags("premium", "code-review")
enterprise_chat = create_chat_with_tags("enterprise", "content-gen")
messages = [HumanMessage(content="Help me with this task")]
response = premium_chat.invoke(messages)
```
#### Tags for Cost Tracking and Analytics
```python
import os
from langchain_openai import ChatOpenAI
from langchain_core.messages import HumanMessage, SystemMessage
# Tags for cost tracking
cost_tracking_chat = ChatOpenAI(
openai_api_base="http://localhost:4000",
model="gpt-4o",
temperature=0.7,
extra_body={
"metadata": {
"tags": [
"cost-center-marketing",
"budget-q4-2024",
"project-launch-campaign",
"high-cost-model" # Flag for expensive models
],
"department": "marketing",
"project_id": "campaign-2024-q4",
"cost_threshold": "high"
}
}
)
messages = [
SystemMessage(content="You are a marketing copywriter."),
HumanMessage(content="Create compelling ad copy for our new product launch.")
]
response = cost_tracking_chat.invoke(messages)
```
#### Tags for A/B Testing
```python
import os
from langchain_openai import ChatOpenAI
from langchain_core.messages import HumanMessage, SystemMessage
import random
def create_ab_test_chat(test_variant: str = None):
"""Create chat instance for A/B testing with appropriate tags"""
if test_variant is None:
test_variant = random.choice(["variant-a", "variant-b"])
return ChatOpenAI(
openai_api_base="http://localhost:4000",
model="gpt-4o",
temperature=0.7 if test_variant == "variant-a" else 0.9, # Different temp for variants
extra_body={
"metadata": {
"tags": [
"ab-test-experiment-1",
f"variant-{test_variant}",
"temperature-test",
"user-experience"
],
"experiment_id": "ab-test-001",
"variant": test_variant,
"test_group": "temperature-optimization"
}
}
)
# Run A/B test
variant_a_chat = create_ab_test_chat("variant-a")
variant_b_chat = create_ab_test_chat("variant-b")
test_message = [HumanMessage(content="Explain quantum computing in simple terms")]
response_a = variant_a_chat.invoke(test_message)
response_b = variant_b_chat.invoke(test_message)
```
### Tag Best Practices
#### 1. **Consistent Naming Convention**
```python
# ✅ Good: Consistent, descriptive tags
tags = ["production", "api-v2", "customer-support", "urgent"]
# ❌ Avoid: Inconsistent or unclear tags
tags = ["prod", "v2", "support", "urgent123"]
```
#### 2. **Hierarchical Tags**
```python
# ✅ Good: Hierarchical structure
tags = ["env:production", "team:backend", "service:api", "priority:high"]
# This allows for easy filtering and grouping
```
#### 3. **Include Context Information**
```python
extra_body={
"metadata": {
"tags": ["production", "user-onboarding"],
"user_id": "user-12345",
"session_id": "session-abc123",
"feature_flag": "new-onboarding-flow",
"environment": "production"
}
}
```
#### 4. **Tag Categories**
Consider organizing tags into categories:
- **Environment**: `production`, `staging`, `development`
- **Team/Service**: `backend`, `frontend`, `api`, `worker`
- **Feature**: `authentication`, `payment`, `notification`
- **Priority**: `critical`, `high`, `medium`, `low`
- **User Type**: `premium`, `enterprise`, `free`
### Using Tags with LiteLLM Proxy
When using tags with LiteLLM Proxy, you can:
1. **Filter requests** based on tags
2. **Track costs** by tags in spend reports
3. **Apply routing rules** based on tags
4. **Monitor usage** with tag-based analytics
#### Example Proxy Configuration with Tags
```yaml
# config.yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: gpt-4o
api_key: your-key
# Tag-based routing rules
tag_routing:
- tags: ["premium", "high-priority"]
models: ["gpt-4o", "claude-3-opus"]
- tags: ["standard"]
models: ["gpt-3.5-turbo", "claude-3-haiku"]
```
### Monitoring and Analytics
Tags enable powerful analytics capabilities:
```python
# Example: Get spend reports by tags
import requests
response = requests.get(
"http://localhost:4000/global/spend/report",
headers={"Authorization": "Bearer sk-your-key"},
params={
"start_date": "2024-01-01",
"end_date": "2024-12-31",
"group_by": "tags"
}
)
spend_by_tags = response.json()
```
This documentation covers the essential patterns for using tags effectively with LangChain and LiteLLM, enabling better organization, tracking, and analytics of your LLM requests.

6
docs/my-website/docs/load_test_rpm.md
View file

@ -53,8 +53,8 @@ model_list = [
},
]
router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, routing_strategy="usage-based-routing-v2", redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
router_2 = Router(model_list=model_list, num_retries=0, routing_strategy="usage-based-routing-v2", enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, routing_strategy="simple-shuffle", redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
router_2 = Router(model_list=model_list, num_retries=0, routing_strategy="simple-shuffle", enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
@ -142,7 +142,7 @@ router_settings:
redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis
redis_password: os.environ/REDIS_PASSWORD
redis_port: os.environ/REDIS_PORT
routing_strategy: usage-based-routing-v2
routing_strategy: simple-shuffle # recommended for best performance
```
### 2. Start proxy 2 instances

62
docs/my-website/docs/mcp.md
View file

@ -40,7 +40,28 @@ LiteLLM supports the following MCP transports:
style={{width: '80%', display: 'block', margin: '0'}}
/>
### Adding a stdio MCP Server
<br/>
<br/>
### Add HTTP MCP Server
This video walks through adding and using an HTTP MCP server on LiteLLM UI and using it in Cursor IDE.
<iframe width="840" height="500" src="https://www.loom.com/embed/e2aebce78e8d46beafeb4bacdde31f14" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
<br/>
<br/>
### Add SSE MCP Server
This video walks through adding and using an SSE MCP server on LiteLLM UI and using it in Cursor IDE.
<iframe width="840" height="500" src="https://www.loom.com/embed/07e04e27f5e74475b9cf8ef8247d2c3e" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
<br/>
<br/>
### Add STDIO MCP Server
For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport type and provide the stdio configuration in JSON format:
@ -92,6 +113,7 @@ mcp_servers:
transport: "http"
description: "My custom MCP server"
auth_type: "api_key"
auth_value: "abc123"
spec_version: "2025-03-26"
```
@ -107,8 +129,42 @@ mcp_servers:
- **Args**: Array of arguments to pass to the command (optional for stdio)
- **Env**: Environment variables to set for the stdio process (optional for stdio)
- **Description**: Optional description for the server
- **Auth Type**: Optional authentication type
- **Spec Version**: Optional MCP specification version (defaults to `2025-03-26`)
- **Auth Type**: Optional authentication type. Supported values:
| Value | Header sent |
|-------|-------------|
| `api_key` | `X-API-Key: <auth_value>` |
| `bearer_token` | `Authorization: Bearer <auth_value>` |
| `basic` | `Authorization: Basic <auth_value>` |
| `authorization` | `Authorization: <auth_value>` |
- **Spec Version**: Optional MCP specification version (defaults to `2025-06-18`)
Examples for each auth type:
```yaml title="MCP auth examples (config.yaml)" showLineNumbers
mcp_servers:
api_key_example:
url: "https://my-mcp-server.com/mcp"
auth_type: "api_key"
auth_value: "abc123" # headers={"X-API-Key": "abc123"}
bearer_example:
url: "https://my-mcp-server.com/mcp"
auth_type: "bearer_token"
auth_value: "abc123" # headers={"Authorization": "Bearer abc123"}
basic_example:
url: "https://my-mcp-server.com/mcp"
auth_type: "basic"
auth_value: "dXNlcjpwYXNz" # headers={"Authorization": "Basic dXNlcjpwYXNz"}
custom_auth_example:
url: "https://my-mcp-server.com/mcp"
auth_type: "authorization"
auth_value: "Token example123" # headers={"Authorization": "Token example123"}
```
### MCP Aliases

19
docs/my-website/docs/observability/braintrust.md
View file

@ -71,6 +71,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
It is recommended that you include the `project_id` or `project_name` to ensure your traces are being written out to the correct Braintrust project.
### Custom Span Names
You can customize the span name in Braintrust logging by passing `span_name` in the metadata. By default, the span name is set to "Chat Completion".
<Tabs>
<TabItem value="sdk" label="SDK">
@ -84,7 +88,9 @@ response = litellm.completion(
"project_id": "1234",
# passing project_name will try to find a project with that name, or create one if it doesn't exist
# if both project_id and project_name are passed, project_id will be used
# "project_name": "my-special-project"
# "project_name": "my-special-project",
# custom span name for this operation (default: "Chat Completion")
"span_name": "User Greeting Handler"
}
)
```
@ -99,6 +105,7 @@ response = litellm.completion(
],
metadata={
"project_id": "1234",
"span_name": "Custom Operation",
"item1": "an item",
"item2": "another item"
}
@ -121,7 +128,8 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
{ "role": "user", "content": "What time is it now? Use your tool"}
],
"metadata": {
"project_id": "my-special-project"
"project_id": "my-special-project",
"span_name": "Tool Usage Request"
}
}'
```
@ -146,7 +154,8 @@ response = client.chat.completions.create(
],
extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params
"metadata": { # 👈 use for logging additional params (e.g. to braintrust)
"project_id": "my-special-project"
"project_id": "my-special-project",
"span_name": "Poetry Generation"
}
}
)
@ -168,3 +177,7 @@ Here's everything you can pass in metadata for a braintrust request
`braintrust_*` - If you are adding metadata from _proxy request headers_, any metadata field starting with `braintrust_` will be passed as metadata to the logging request. If you are using the SDK, just pass your metadata like normal (e.g., `metadata={"project_name": "my-test-project", "item1": "an item", "item2": "another item"}`)
`project_id` - Set the project id for a braintrust call. Default is `litellm`.
`project_name` - Set the project name for a braintrust call. Will try to find a project with that name, or create one if it doesn't exist. If both `project_id` and `project_name` are passed, `project_id` will be used.
`span_name` - Set a custom span name for the operation. Default is `"Chat Completion"`. Use this to provide more descriptive names for different types of operations in your application (e.g., "User Query", "Document Summary", "Code Generation").

5
docs/my-website/docs/observability/callbacks.md
View file

@ -4,9 +4,14 @@
liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses.
:::tip
**New to LiteLLM Callbacks?** Check out our comprehensive [Callback Management Guide](./callback_management.md) to understand when to use different callback hooks like `async_log_success_event` vs `async_post_call_success_hook`.
:::
liteLLM supports:
- [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback)
- [Callback Management Guide](./callback_management.md) - **Comprehensive guide for choosing the right hooks**
- [Lunary](https://lunary.ai/docs)
- [Langfuse](https://langfuse.com/docs)
- [LangSmith](https://www.langchain.com/langsmith)

209
docs/my-website/docs/observability/cloudzero.md Normal file
View file

@ -0,0 +1,209 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# CloudZero Integration
LiteLLM provides an integration with CloudZero's AnyCost API, allowing you to export your LLM usage data to CloudZero for cost tracking analysis.
## Overview
| Property | Details |
|----------|---------|
| Description | Export LiteLLM usage data to CloudZero AnyCost API for cost tracking and analysis |
| callback name | `cloudzero`|
| Supported Operations | • Automatic hourly data export<br/>• Manual data export<br/>• Dry run testing<br/>• Cost and token usage tracking |
| Data Format | CloudZero Billing Format (CBF) with proper resource tagging |
| Export Frequency | Hourly (configurable via `CLOUDZERO_EXPORT_INTERVAL_MINUTES`) |
## Environment Variables
| Variable | Required | Description | Example |
|----------|----------|-------------|---------|
| `CLOUDZERO_API_KEY` | Yes | Your CloudZero API key | `cz_api_xxxxxxxxxx` |
| `CLOUDZERO_CONNECTION_ID` | Yes | CloudZero connection ID for data submission | `conn_xxxxxxxxxx` |
| `CLOUDZERO_TIMEZONE` | No | Timezone for date handling (default: UTC) | `America/New_York` |
| `CLOUDZERO_EXPORT_INTERVAL_MINUTES` | No | Export frequency in minutes (default: 60) | `60` |
## Setup
### End to End Video Walkthrough
This video walks through the entire process of setting up LiteLLM with CloudZero integration and viewing LiteLLM exported usage data in CloudZero.
<iframe width="840" height="500" src="https://www.loom.com/embed/59b57593183f4cc3b1c05a2dd3277f92" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
### Step 1: Configure Environment Variables
Set your CloudZero credentials in your environment:
```bash
export CLOUDZERO_API_KEY="cz_api_xxxxxxxxxx"
export CLOUDZERO_CONNECTION_ID="conn_xxxxxxxxxx"
export CLOUDZERO_TIMEZONE="UTC" # Optional, defaults to UTC
```
### Step 2: Enable CloudZero Integration
Add the CloudZero callback to your LiteLLM configuration YAML file:
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-xxxxxxx
litellm_settings:
callbacks: ["cloudzero"] # Enable CloudZero integration
```
### Step 3: Start LiteLLM Proxy
Start your LiteLLM proxy with the configuration:
```bash
litellm --config /path/to/config.yaml
```
## Testing Your Setup
### Dry Run Export
Call the dry run endpoint to test your CloudZero configuration without sending data to CloudZero. This endpoint will not send any data to CloudZero, but will return the data that would be exported.
```bash
curl -X POST "http://localhost:4000/cloudzero/dry-run" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"limit": 10
}' | jq
```
**Expected Response:**
```json
{
"message": "CloudZero dry run export completed successfully.",
"status": "success",
"dry_run_data": {
"usage_data": [...],
"cbf_data": [...],
"summary": {
"total_cost": 0.05,
"total_tokens": 1250,
"total_records": 10
}
}
}
```
### Manual Export
Call the export endpoint to send data immediately to CloudZero. We suggest setting a small `limit` to test the export. This will only export the last 10 records to CloudZero. Note: Cloudzero can take up to 15 minutes to process the exported data.
```bash
curl -X POST "http://localhost:4000/cloudzero/export" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"limit": 10
}' | jq
```
**Expected Response:**
```json
{
"message": "CloudZero export completed successfully",
"status": "success"
}
```
## Data Export Details
### Automatic Export Schedule
- **Frequency**: Every 60 minutes (configurable via `CLOUDZERO_EXPORT_INTERVAL_MINUTES`)
- **Data Processing**: LiteLLM automatically processes and exports usage data hourly
- **CloudZero Processing**: CloudZero typically takes 10-15 minutes to process data from LiteLLM
### Data Format
LiteLLM exports data in CloudZero Billing Format (CBF) with the following structure:
```json
{
"time/usage_start": "2024-01-15T14:00:00Z",
"cost/cost": 0.002,
"usage/amount": 150,
"usage/units": "tokens",
"resource/id": "czrn:litellm:openai:cross-region:team-123:llm-usage:gpt-4o",
"resource/service": "litellm",
"resource/account": "team-123",
"resource/region": "cross-region",
"resource/usage_family": "llm-usage",
"resource/tag:provider": "openai",
"resource/tag:model": "gpt-4o",
"resource/tag:prompt_tokens": "100",
"resource/tag:completion_tokens": "50"
}
```
### Resource Tagging
LiteLLM automatically creates comprehensive resource tags for cost attribution:
- **Provider Tags**: `openai`, `anthropic`, `azure`, etc.
- **Model Tags**: Specific model names like `gpt-4o`, `claude-3-sonnet`
- **Team/User Tags**: Team IDs and user IDs for cost allocation
- **Token Breakdown**: Separate tracking of prompt and completion tokens
- **Usage Metrics**: Total tokens consumed per request
## Advanced Configuration
### Custom Export Frequency
Change the export frequency (not recommended to go below 60 minutes):
```bash
export CLOUDZERO_EXPORT_INTERVAL_MINUTES=120 # Export every 2 hours
```
### Custom Time Range Export
Export data for a specific time range:
```bash
curl -X POST "http://localhost:4000/cloudzero/export" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"start_time_utc": "2024-01-15T00:00:00Z",
"end_time_utc": "2024-01-15T23:59:59Z",
"operation": "replace_hourly"
}' | jq
```
## Troubleshooting
### Common Issues
1. **Missing Credentials Error**
```
CloudZero configuration missing. Please set CLOUDZERO_API_KEY and CLOUDZERO_CONNECTION_ID environment variables.
```
**Solution**: Ensure both environment variables are set with valid values.
2. **Connection Issues**
- Verify your CloudZero API key is valid
- Check that the connection ID exists in your CloudZero account
- Ensure your proxy has internet access to reach CloudZero's API
3. **No Data in CloudZero**
- CloudZero can take 10-15 minutes to process data
- Check that your LiteLLM proxy is generating usage data
- Use the dry-run endpoint to verify data is being formatted correctly
## Related Links
- [CloudZero Documentation](https://docs.cloudzero.com/)
- [CloudZero AnyCost API](https://docs.cloudzero.com/reference/anycost-api)

299
docs/my-website/docs/observability/custom_callback.md
View file

@ -4,7 +4,6 @@
**For PROXY** [Go Here](../proxy/logging.md#custom-callback-class-async)
:::
## Callback Class
You can create a custom callback class to precisely log events as they occur in litellm.
@ -57,6 +56,17 @@ def async completion():
asyncio.run(completion())
```
## Common Hooks
- `async_log_success_event` - Log successful API calls
- `async_log_failure_event` - Log failed API calls
- `log_pre_api_call` - Log before API call
- `log_post_api_call` - Log after API call
**Proxy-only hooks** (only work with LiteLLM Proxy):
- `async_post_call_success_hook` - Access user data + modify responses
- `async_pre_call_hook` - Modify requests before sending
## Callback Functions
If you just want to log on a specific event (e.g. on input) - you can use callback functions.
@ -174,260 +184,87 @@ async def test_chat_openai():
asyncio.run(test_chat_openai())
```
:::info
## What's Available in kwargs?
We're actively trying to expand this to other event types. [Tell us if you need this!](https://github.com/BerriAI/litellm/issues/1007)
:::
## What's in kwargs?
Notice we pass in a kwargs argument to custom callback.
```python
def custom_callback(
kwargs, # kwargs to completion
completion_response, # response from completion
start_time, end_time # start/end time
):
# Your custom code here
print("LITELLM: in custom callback function")
print("kwargs", kwargs)
print("completion_response", completion_response)
print("start_time", start_time)
print("end_time", end_time)
```
This is a dictionary containing all the model-call details (the params we receive, the values we send to the http endpoint, the response we receive, stacktrace in case of errors, etc.).
This is all logged in the [model_call_details via our Logger](https://github.com/BerriAI/litellm/blob/fc757dc1b47d2eb9d0ea47d6ad224955b705059d/litellm/utils.py#L246).
Here's exactly what you can expect in the kwargs dictionary:
```shell
### DEFAULT PARAMS ###
"model": self.model,
"messages": self.messages,
"optional_params": self.optional_params, # model-specific params passed in
"litellm_params": self.litellm_params, # litellm-specific params passed in (e.g. metadata passed to completion call)
"start_time": self.start_time, # datetime object of when call was started
### PRE-API CALL PARAMS ### (check via kwargs["log_event_type"]="pre_api_call")
"input" = input # the exact prompt sent to the LLM API
"api_key" = api_key # the api key used for that LLM API
"additional_args" = additional_args # any additional details for that API call (e.g. contains optional params sent)
### POST-API CALL PARAMS ### (check via kwargs["log_event_type"]="post_api_call")
"original_response" = original_response # the original http response received (saved via response.text)
### ON-SUCCESS PARAMS ### (check via kwargs["log_event_type"]="successful_api_call")
"complete_streaming_response" = complete_streaming_response # the complete streamed response (only set if `completion(..stream=True)`)
"end_time" = end_time # datetime object of when call was completed
### ON-FAILURE PARAMS ### (check via kwargs["log_event_type"]="failed_api_call")
"exception" = exception # the Exception raised
"traceback_exception" = traceback_exception # the traceback generated via `traceback.format_exc()`
"end_time" = end_time # datetime object of when call was completed
```
### Cache hits
Cache hits are logged in success events as `kwarg["cache_hit"]`.
Here's an example of accessing it:
```python
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm import completion, acompletion, Cache
class MyCustomHandler(CustomLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
print(f"On Success")
print(f"Value of Cache hit: {kwargs['cache_hit']"})
async def test_async_completion_azure_caching():
customHandler_caching = MyCustomHandler()
litellm.cache = Cache(type="redis", host=os.environ['REDIS_HOST'], port=os.environ['REDIS_PORT'], password=os.environ['REDIS_PASSWORD'])
litellm.callbacks = [customHandler_caching]
unique_time = time.time()
response1 = await litellm.acompletion(model="azure/chatgpt-v-2",
messages=[{
"role": "user",
"content": f"Hi 👋 - i'm async azure {unique_time}"
}],
caching=True)
await asyncio.sleep(1)
print(f"customHandler_caching.states pre-cache hit: {customHandler_caching.states}")
response2 = await litellm.acompletion(model="azure/chatgpt-v-2",
messages=[{
"role": "user",
"content": f"Hi 👋 - i'm async azure {unique_time}"
}],
caching=True)
await asyncio.sleep(1) # success callbacks are done in parallel
print(f"customHandler_caching.states post-cache hit: {customHandler_caching.states}")
assert len(customHandler_caching.errors) == 0
assert len(customHandler_caching.states) == 4 # pre, post, success, success
```
### Get complete streaming response
LiteLLM will pass you the complete streaming response in the final streaming chunk as part of the kwargs for your custom callback function.
The kwargs dictionary contains all the details about your API call:
```python
# litellm.set_verbose = False
def custom_callback(
kwargs, # kwargs to completion
completion_response, # response from completion
start_time, end_time # start/end time
):
# print(f"streaming response: {completion_response}")
if "complete_streaming_response" in kwargs:
print(f"Complete Streaming Response: {kwargs['complete_streaming_response']}")
# Assign the custom callback function
litellm.success_callback = [custom_callback]
response = completion(model="claude-instant-1", messages=messages, stream=True)
for idx, chunk in enumerate(response):
pass
```
### Log additional metadata
LiteLLM accepts a metadata dictionary in the completion call. You can pass additional metadata into your completion call via `completion(..., metadata={"key": "value"})`.
Since this is a [litellm-specific param](https://github.com/BerriAI/litellm/blob/b6a015404eed8a0fa701e98f4581604629300ee3/litellm/main.py#L235), it's accessible via kwargs["litellm_params"]
```python
from litellm import completion
import os, litellm
## set ENV variables
os.environ["OPENAI_API_KEY"] = "your-api-key"
messages = [{ "content": "Hello, how are you?","role": "user"}]
def custom_callback(
kwargs, # kwargs to completion
completion_response, # response from completion
start_time, end_time # start/end time
):
print(kwargs["litellm_params"]["metadata"])
def custom_callback(kwargs, completion_response, start_time, end_time):
# Access common data
model = kwargs.get("model")
messages = kwargs.get("messages", [])
cost = kwargs.get("response_cost", 0)
cache_hit = kwargs.get("cache_hit", False)
# Assign the custom callback function
litellm.success_callback = [custom_callback]
response = litellm.completion(model="gpt-3.5-turbo", messages=messages, metadata={"hello": "world"})
# Access metadata you passed in
metadata = kwargs.get("litellm_params", {}).get("metadata", {})
```
## Examples
**Key fields in kwargs:**
- `model` - The model name
- `messages` - Input messages
- `response_cost` - Calculated cost
- `cache_hit` - Whether response was cached
- `litellm_params.metadata` - Your custom metadata
### Custom Callback to track costs for Streaming + Non-Streaming
By default, the response cost is accessible in the logging object via `kwargs["response_cost"]` on success (sync + async)
## Practical Examples
### Track API Costs
```python
def track_cost_callback(kwargs, completion_response, start_time, end_time):
cost = kwargs["response_cost"] # litellm calculates this for you
print(f"Request cost: ${cost}")
# Step 1. Write your custom callback function
def track_cost_callback(
kwargs, # kwargs to completion
completion_response, # response from completion
start_time, end_time # start/end time
):
try:
response_cost = kwargs["response_cost"] # litellm calculates response cost for you
print("regular response_cost", response_cost)
except:
pass
# Step 2. Assign the custom callback function
litellm.success_callback = [track_cost_callback]
# Step 3. Make litellm.completion call
response = completion(
model="gpt-3.5-turbo",
messages=[
{
"role": "user",
"content": "Hi 👋 - i'm openai"
}
]
)
print(response)
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello"}])
```
### Custom Callback to log transformed Input to LLMs
### Log Inputs to LLMs
```python
def get_transformed_inputs(
kwargs,
):
def get_transformed_inputs(kwargs):
params_to_model = kwargs["additional_args"]["complete_input_dict"]
print("params to model", params_to_model)
litellm.input_callback = [get_transformed_inputs]
def test_chat_openai():
try:
response = completion(model="claude-2",
messages=[{
"role": "user",
"content": "Hi 👋 - i'm openai"
}])
print(response)
except Exception as e:
print(e)
pass
response = completion(model="claude-2", messages=[{"role": "user", "content": "Hello"}])
```
#### Output
```shell
params to model {'model': 'claude-2', 'prompt': "\n\nHuman: Hi 👋 - i'm openai\n\nAssistant: ", 'max_tokens_to_sample': 256}
### Send to External Service
```python
import requests
def send_to_analytics(kwargs, completion_response, start_time, end_time):
data = {
"model": kwargs.get("model"),
"cost": kwargs.get("response_cost", 0),
"duration": (end_time - start_time).total_seconds()
}
requests.post("https://your-analytics.com/api", json=data)
litellm.success_callback = [send_to_analytics]
```
### Custom Callback to write to Mixpanel
## Common Issues
### Callback Not Called
Make sure you:
1. Register callbacks correctly: `litellm.callbacks = [MyHandler()]`
2. Use the right hook names (check spelling)
3. Don't use proxy-only hooks in library mode
### Performance Issues
- Use async hooks for I/O operations
- Don't block in callback functions
- Handle exceptions properly:
```python
import mixpanel
import litellm
from litellm import completion
def custom_callback(
kwargs, # kwargs to completion
completion_response, # response from completion
start_time, end_time # start/end time
):
# Your custom code here
mixpanel.track("LLM Response", {"llm_response": completion_response})
# Assign the custom callback function
litellm.success_callback = [custom_callback]
response = completion(
model="gpt-3.5-turbo",
messages=[
{
"role": "user",
"content": "Hi 👋 - i'm openai"
}
]
)
print(response)
class SafeHandler(CustomLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
await external_service(response_obj)
except Exception as e:
print(f"Callback error: {e}") # Log but don't break the flow
```

23
docs/my-website/docs/observability/langfuse_otel_integration.md
View file

@ -35,14 +35,14 @@ The Langfuse OpenTelemetry integration allows you to send LiteLLM traces and obs
|----------|----------|-------------|---------|
| `LANGFUSE_PUBLIC_KEY` | Yes | Your Langfuse public key | `pk-lf-...` |
| `LANGFUSE_SECRET_KEY` | Yes | Your Langfuse secret key | `sk-lf-...` |
| `LANGFUSE_HOST` | No | Langfuse host URL | `https://us.cloud.langfuse.com` (default) |
| `LANGFUSE_OTEL_HOST` | No | OTEL endpoint host | `https://otel.my-langfuse.com` |
### Endpoint Resolution
The integration automatically constructs the OTEL endpoint from the `LANGFUSE_HOST`:
The integration automatically constructs the OTEL endpoint from `LANGFUSE_OTEL_HOST`
- **Default (US)**: `https://us.cloud.langfuse.com/api/public/otel`
- **EU Region**: `https://cloud.langfuse.com/api/public/otel`
- **Self-hosted**: `{LANGFUSE_HOST}/api/public/otel`
- **Self-hosted**: `{LANGFUSE_OTEL_HOST}/api/public/otel`
## Usage
@ -77,11 +77,11 @@ os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
# Use EU region
os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # US region (default)
os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint
# Or use self-hosted instance
# os.environ["LANGFUSE_HOST"] = "https://my-langfuse.company.com"
# os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com"
litellm.callbacks = ["langfuse_otel"]
```
@ -98,14 +98,16 @@ import litellm
# Get keys for your project from the project settings page: https://cloud.langfuse.com
os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # US region
os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_OTEL_HOST"] = "https://us.cloud.langfuse.com" # US region
# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint
LANGFUSE_AUTH = base64.b64encode(
f"{os.environ.get('LANGFUSE_PUBLIC_KEY')}:{os.environ.get('LANGFUSE_SECRET_KEY')}".encode()
).decode()
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = os.environ.get("LANGFUSE_HOST") + "/api/public/otel"
host = os.environ.get("LANGFUSE_OTEL_HOST")
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = host + "/api/public/otel"
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"Authorization=Basic {LANGFUSE_AUTH}"
litellm.callbacks = ["langfuse_otel"]
@ -120,7 +122,8 @@ Add the integration to your proxy configuration:
```bash
export LANGFUSE_PUBLIC_KEY="pk-lf-..."
export LANGFUSE_SECRET_KEY="sk-lf-..."
export LANGFUSE_HOST="https://us.cloud.langfuse.com" # Default US region
export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com" # Default US region
# export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com" # custom OTEL endpoint
```
2. Setup config.yaml

7
docs/my-website/docs/pass_through/google_ai_studio.md
View file

@ -230,6 +230,13 @@ curl -X POST "https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5
```
## **Example 4: Video Generation with Veo**
Generate videos using Google's Veo model through LiteLLM pass-through routes.
[**→ Complete Veo Video Generation Guide**](../proxy/veo_video_generation.md)
## Advanced
Pre-requisites

40
docs/my-website/docs/pass_through/intro.md
View file

@ -11,3 +11,43 @@ These endpoints are useful for 2 scenarios:
## How is your request handled?
The request is passed through to the provider's endpoint. The response is then passed back to the client. **No translation is done.**
### Request Forwarding Process
1. **Request Reception**: LiteLLM receives your request at `/provider/endpoint`
2. **Authentication**: Your LiteLLM API key is validated and mapped to the provider's API key
3. **Request Transformation**: Request is reformatted for the target provider's API
4. **Forwarding**: Request is sent to the actual provider endpoint
5. **Response Handling**: Provider response is returned directly to you
### Authentication Flow
```mermaid
graph LR
A[Client Request] --> B[LiteLLM Proxy]
B --> C[Validate LiteLLM API Key]
C --> D[Map to Provider API Key]
D --> E[Forward to Provider]
E --> F[Return Response]
```
**Key Points:**
- Use your **LiteLLM API key** in requests, not the provider's key
- LiteLLM handles the provider authentication internally
- Same authentication works across all passthrough endpoints
### Error Handling
**Provider Errors**: Forwarded directly to you with original error codes and messages
**LiteLLM Errors**:
- `401`: Invalid LiteLLM API key
- `404`: Provider or endpoint not supported
- `500`: Internal routing/forwarding errors
### Benefits
- **Unified Authentication**: One API key for all providers
- **Centralized Logging**: All requests logged through LiteLLM
- **Cost Tracking**: Usage tracked across all endpoints
- **Access Control**: Same permissions apply to passthrough endpoints

30
docs/my-website/docs/providers/aiml.md
View file

@ -1,5 +1,23 @@
# AI/ML API
https://aimlapi.com/
## Overview
| Property | Details |
|-------|-------|
| Description | AI/ML API provides access to state-of-the-art AI models including flux-pro/v1.1 for high-quality image generation. |
| Provider Route on LiteLLM | `aiml/` |
| Link to Provider Doc | [AI/ML API ↗](https://docs.aimlapi.com/) |
| Supported Operations | [`/chat/completions`], [`/images/generations`](#image-generation) |
LiteLLM supports AI/ML API Image Generation calls.
## API Base, Key
```python
# env variable
os.environ['AIML_API_KEY'] = "your-api-key"
os.environ['AIML_API_BASE'] = "https://api.aimlapi.com" # [optional]
```
Getting started with the AI/ML API is simple. Follow these steps to set up your integration:
### 1. Get Your API Key
@ -24,7 +42,7 @@ You can choose from LLama, Qwen, Flux, and 200+ other open and closed-source mod
import litellm
response = litellm.completion(
model="openai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@ -42,7 +60,7 @@ response = litellm.completion(
import litellm
response = litellm.completion(
model="openai/Qwen/Qwen2-72B-Instruct", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/Qwen/Qwen2-72B-Instruct", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@ -67,7 +85,7 @@ import litellm
async def main():
response = await litellm.acompletion(
model="openai/anthropic/claude-3-5-haiku", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/anthropic/claude-3-5-haiku", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@ -97,7 +115,7 @@ async def main():
try:
print("test acompletion + streaming")
response = await litellm.acompletion(
model="openai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[{"content": "Hey, how's it going?", "role": "user"}],
@ -125,7 +143,7 @@ import litellm
async def main():
response = await litellm.aembedding(
model="openai/text-embedding-3-small", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/text-embedding-3-small", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v1", # 👈 the URL has changed from v2 to v1
input="Your text string",
@ -147,7 +165,7 @@ import litellm
async def main():
response = await litellm.aimage_generation(
model="openai/dall-e-3", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/dall-e-3", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v1", # 👈 the URL has changed from v2 to v1
prompt="A cute baby sea otter",

21
docs/my-website/docs/providers/anthropic.md
View file

@ -55,8 +55,29 @@ import os
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
# os.environ["ANTHROPIC_API_BASE"] = "" # [OPTIONAL] or 'ANTHROPIC_BASE_URL'
# os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # [OPTIONAL] Disable automatic URL suffix appending
```
### Custom API Base
When using a custom API base for Anthropic (e.g., a proxy or custom endpoint), LiteLLM automatically appends the appropriate suffix (`/v1/messages` or `/v1/complete`) to your base URL.
If your custom endpoint already includes the full path or doesn't follow Anthropic's standard URL structure, you can disable this automatic suffix appending:
```python
import os
os.environ["ANTHROPIC_API_BASE"] = "https://my-custom-endpoint.com/custom/path"
os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # Prevents automatic suffix
```
Without `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX`:
- Base URL `https://my-proxy.com` → `https://my-proxy.com/v1/messages`
- Base URL `https://my-proxy.com/api` → `https://my-proxy.com/api/v1/messages`
With `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX=true`:
- Base URL `https://my-proxy.com/custom/path` → `https://my-proxy.com/custom/path` (unchanged)
## Usage
```python

2
docs/my-website/docs/providers/bedrock.md
View file

@ -467,7 +467,7 @@ print(f"\nResponse: {resp}")
## Usage - 'thinking' / 'reasoning content'
This is currently only supported for Anthropic's Claude 3.7 Sonnet + Deepseek R1.
This is currently only supported for Anthropic's Claude 3.7 Sonnet + Deepseek R1 + GPT-OSS models.
Works on v1.61.20+.

144
docs/my-website/docs/providers/cometapi.md Normal file
View file

@ -0,0 +1,144 @@
# CometAPI
LiteLLM supports all AI models from [CometAPI](https://www.cometapi.com/). CometAPI provides access to 500+ AI models through a unified API interface, including cutting-edge models like GPT-5, Claude Opus 4.1, and various other state-of-the-art language models.
## Authentication
To use CometAPI models, you need to obtain an API key from [CometAPI Token Console](https://api.cometapi.com/console/token). CometAPI offers free tokens for new users - you can get your free API key instantly by registering.
## Usage
Set your CometAPI key as an environment variable and use the completion function:
```python
import os
from litellm import completion
# Set API key
os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
# Define messages
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Method 1: Using environment variable (recommended)
response = completion(
model="cometapi/gpt-5",
messages=messages
)
print(response.choices[0].message.content)
```
### Alternative Usage - Explicit API Key
You can also pass the API key explicitly:
```python
import os
from litellm import completion
# Define messages
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Method 2: Explicitly passing API key
response = completion(
model="cometapi/gpt-4o",
messages=messages,
api_key="your_comet_api_key_here"
)
print(response.choices[0].message.content)
```
## Usage - Streaming
Just set `stream=True` when calling completion:
```python
import os
from litellm import completion
os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
messages = [{"content": "Hello, how are you?", "role": "user"}]
response = completion(
model="cometapi/gpt-5",
messages=messages,
stream=True
)
for chunk in response:
print(chunk.choices[0].delta.content or "", end="")
```
## Usage - Async Streaming
For async streaming, use `acompletion`:
```python
from litellm import acompletion
import asyncio, os, traceback
async def completion_call():
try:
os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
print("test acompletion + streaming")
response = await acompletion(
model="cometapi/chatgpt-4o-latest",
messages=[{"content": "Hello, how are you?", "role": "user"}],
stream=True
)
print(f"response: {response}")
async for chunk in response:
print(chunk)
except:
print(f"error occurred: {traceback.format_exc()}")
pass
# Run the async function
await completion_call()
```
## CometAPI Models
CometAPI offers access to 500+ AI models through a unified API. Some popular models include:
| Model Name | Function Call |
|------------|---------------|
| cometapi/gpt-5 | `completion('cometapi/gpt-5', messages)` |
| cometapi/gpt-5-mini | `completion('cometapi/gpt-5-mini', messages)` |
| cometapi/gpt-5-nano | `completion('cometapi/gpt-5-nano', messages)` |
| cometapi/gpt-oss-20b | `completion('cometapi/gpt-oss-20b', messages)` |
| cometapi/gpt-oss-120b | `completion('cometapi/gpt-oss-120b', messages)` |
| cometapi/chatgpt-4o-latest | `completion('cometapi/chatgpt-4o-latest', messages)` |
For a complete list of available models, visit the [CometAPI Models page](https://www.cometapi.com/model/).
## Environment Variables
| Variable | Description | Required |
|----------|-------------|----------|
| `COMETAPI_KEY` | Your CometAPI API key | Yes |
## Error Handling
```python
import os
from litellm import completion
try:
os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
messages = [{"content": "Hello, how are you?", "role": "user"}]
response = completion(
model="cometapi/gpt-5",
messages=messages
)
print(response.choices[0].message.content)
except Exception as e:
print(f"Error: {e}")
```

5
docs/my-website/docs/providers/databricks.md
View file

@ -282,6 +282,11 @@ ModelResponse(
)
```
### Citations
Anthropic models served through Databricks can return citation metadata. LiteLLM
exposes these via `response.choices[0].message.provider_specific_fields["citations"]`.
### Pass `thinking` to Anthropic models
You can also pass the `thinking` parameter to Anthropic models.

43
docs/my-website/docs/providers/datarobot.md Normal file
View file

@ -0,0 +1,43 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# DataRobot
LiteLLM supports all models from [DataRobot](https://datarobot.com). Select `datarobot` as the provider to route your request through the `datarobot` OpenAI-compatible endpoint using the upstream [official OpenAI Python API library](https://github.com/openai/openai-python/blob/main/README.md).
## Usage
### Environment variables
```python
import os
from litellm import completion
os.environ["DATAROBOT_API_KEY"] = ""
os.environ["DATAROBOT_API_BASE"] = "" # [OPTIONAL] defaults to https://app.datarobot.com
response = completion(
model="datarobot/openai/gpt-4o-mini",
messages=messages,
)
### Completion
```python
import litellm
import os
response = litellm.completion(
model="datarobot/openai/gpt-4o-mini", # add `datarobot/` prefix to model so litellm knows to route through DataRobot
messages=[
{
"role": "user",
"content": "Hey, how's it going?",
}
],
)
print(response)
```
## DataRobot completion models
🚨 LiteLLM supports _all_ DataRobot LLM gateway models. To get a list for your installation and user account, send the following CURL command:
`curl -X GET -H "Authorization: Bearer $DATAROBOT_API_TOKEN" "$DATAROBOT_ENDPOINT/genai/llmgw/catalog/" | jq | grep 'model":'DATAROBOT_ENDPOINT/genai/llmgw/catalog/`

140
docs/my-website/docs/providers/deepinfra.md
View file

@ -1,3 +1,6 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# DeepInfra
https://deepinfra.com/
@ -7,6 +10,11 @@ https://deepinfra.com/
:::
## Table of Contents
- [API Key](#api-key)
- [Chat Models](#chat-models)
- [Rerank Endpoint](#rerank-endpoint)
## API Key
```python
@ -53,3 +61,135 @@ for chunk in response:
| codellama/CodeLlama-34b-Instruct-hf | `completion(model="deepinfra/codellama/CodeLlama-34b-Instruct-hf", messages)` |
| mistralai/Mistral-7B-Instruct-v0.1 | `completion(model="deepinfra/mistralai/Mistral-7B-Instruct-v0.1", messages)` |
| jondurbin/airoboros-l2-70b-gpt4-1.4.1 | `completion(model="deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1", messages)` |
## Rerank Endpoint
LiteLLM provides a Cohere API compatible `/rerank` endpoint for DeepInfra rerank models.
### Supported Rerank Models
| Model Name | Description |
|------------|-------------|
| `deepinfra/Qwen/Qwen3-Reranker-0.6B` | Lightweight rerank model (0.6B parameters) |
| `deepinfra/Qwen/Qwen3-Reranker-4B` | Medium rerank model (4B parameters) |
| `deepinfra/Qwen/Qwen3-Reranker-8B` | Large rerank model (8B parameters) |
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import rerank
import os
os.environ["DEEPINFRA_API_KEY"] = "your-api-key"
response = rerank(
model="deepinfra/Qwen/Qwen3-Reranker-0.6B",
query="What is the capital of France?",
documents=[
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
]
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Add to config.yaml
```yaml
model_list:
- model_name: Qwen/Qwen3-Reranker-0.6B
litellm_params:
model: deepinfra/Qwen/Qwen3-Reranker-0.6B
api_key: os.environ/DEEPINFRA_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000/
```
3. Test it!
```bash
curl -L -X POST 'http://0.0.0.0:4000/rerank' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "Qwen/Qwen3-Reranker-0.6B",
"query": "What is the capital of France?",
"documents": [
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
]
}'
```
</TabItem>
</Tabs>
### Supported Cohere Rerank API Params
| Param | Type | Description |
| ------------------ | ----------- | ----------------------------------------------- |
| `query` | `str` | The query to rerank the documents against |
| `documents` | `list[str]` | The documents to rerank |
### Provider-specific parameters
Pass any deepinfra specific parameters as a keyword argument to the rerank function, e.g.
```
response = rerank(
model="deepinfra/Qwen/Qwen3-Reranker-0.6B",
query="What is the capital of France?",
documents=[
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
],
my_custom_param="my_custom_value", # any other deepinfra specific parameters
)
```
### Response Format
```json
{
"id": "request-id",
"results": [
{
"index": 0,
"relevance_score": 0.9975274205207825
},
{
"index": 1,
"relevance_score": 0.011687257327139378
}
],
"meta": {
"billed_units": {
"total_tokens": 427
},
"tokens": {
"input_tokens": 427,
"output_tokens": 0
}
}
}
```

10
docs/my-website/docs/providers/google_ai_studio/image_gen.md
View file

@ -42,7 +42,7 @@ os.environ["GEMINI_API_KEY"] = "your-api-key-here"
# Generate a single image
response = litellm.image_generation(
model="gemini/imagen-4.0-generate-preview-06-06",
model="gemini/imagen-4.0-generate-001",
prompt="A cute baby sea otter swimming in crystal clear water"
)
@ -64,7 +64,7 @@ async def generate_image():
# Generate image asynchronously
response = await litellm.aimage_generation(
model="gemini/imagen-4.0-generate-preview-06-06",
model="gemini/imagen-4.0-generate-001",
prompt="A beautiful sunset over mountains with vibrant colors",
n=1,
)
@ -89,7 +89,7 @@ os.environ["GEMINI_API_KEY"] = "your-api-key-here"
# Generate image with additional parameters
response = litellm.image_generation(
model="gemini/imagen-4.0-generate-preview-06-06",
model="gemini/imagen-4.0-generate-001",
prompt="A futuristic cityscape at night with neon lights",
n=1,
size="1024x1024",
@ -112,7 +112,7 @@ for image in response.data:
model_list:
- model_name: google-imagen
litellm_params:
model: gemini/imagen-4.0-generate-preview-06-06
model: gemini/imagen-4.0-generate-001
api_key: os.environ/GEMINI_API_KEY
model_info:
mode: image_generation
@ -198,7 +198,7 @@ Google AI Studio Image Generation supports the following OpenAI-compatible param
| Parameter | Type | Description | Default | Example |
|-----------|------|-------------|---------|---------|
| `prompt` | string | Text description of the image to generate | Required | `"A sunset over the ocean"` |
| `model` | string | The model to use for generation | Required | `"gemini/imagen-4.0-generate-preview-06-06"` |
| `model` | string | The model to use for generation | Required | `"gemini/imagen-4.0-generate-001"` |
| `n` | integer | Number of images to generate (1-4) | `1` | `2` |
| `size` | string | Image dimensions | `"1024x1024"` | `"512x512"`, `"1024x1024"` |

76
docs/my-website/docs/providers/heroku.md Normal file
View file

@ -0,0 +1,76 @@
# Heroku
## Provision a Model
To use Heroku with LiteLLM, [configure a Heroku app and attach a supported model](https://devcenter.heroku.com/articles/heroku-inference#provision-access-to-an-ai-model-resource).
## Supported Models
Heroku for LiteLLM supports various [chat](https://devcenter.heroku.com/articles/heroku-inference-api-v1-chat-completions) models:
| Model | Region |
|-----------------------------------|---------|
| [`heroku/claude-sonnet-4`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-4-sonnet) | US, EU |
| [`heroku/claude-3-7-sonnet`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-7-sonnet) | US, EU |
| [`heroku/claude-3-5-sonnet-latest`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-5-sonnet-latest) | US |
| [`heroku/claude-3-5-haiku`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-5-haiku) | US |
| [`heroku/claude-3`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-haiku) | EU |
## Environment Variables
When you attach a model to a Heroku app, three config variables are set:
- `INFERENCE_KEY`: The API key used for authenticating requests to the model.
- `INFERENCE_MODEL_ID`: The name of the model, for example`claude-3-5-haiku`.
- `INFERENCE_URL`: The base URL for calling the model.
Both `INFERENCE_KEY` and `INFERENCE_URL` are required to make calls to your model.
For more information on these variables, see the [Heroku documentation](https://devcenter.heroku.com/articles/heroku-inference#model-resource-config-vars).
## Usage Examples
### Using Config Variables
Heroku uses the following LiteLLM API config variables:
- `HEROKU_API_KEY`: This value corresponds to [LiteLLM's `api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this variable to the value of Heroku's `INFERENCE_KEY` config variable.
- `HEROKU_API_BASE`: This value corresponds to [LiteLLM's `api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this variable to the value of Heroku's `INFERENCE_URL` config variable.
In this example, we don't explicitly pass the `api_key` and `api_base` variables. Instead, we set the config variables which Heroku will use:
```python
import os
from litellm import completion
os.environ["HEROKU_API_BASE"] = "https://us.inference.heroku.com"
os.environ["HEROKU_API_KEY"] = "fake-heroku-key"
response = completion(
model="heroku/claude-3-5-haiku",
messages=[
{"role": "user", "content": "write code for saying hey from LiteLLM"}
]
)
print(response)
```
> Include the `heroku/` prefix in the model name so LiteLLM knows the model provider to use.
### Explicitly Setting `api_key` and `api_base`
```python
from litellm import completion
response = completion(
model="heroku/claude-sonnet-4",
api_key="fake-heroku-key",
api_base="https://us.inference.heroku.com",
messages=[
{"role": "user", "content": "write code for saying hey from LiteLLM"}
],
)
```
> Include the `heroku/` prefix in the model name so LiteLLM knows the model provider to use.

8
docs/my-website/docs/providers/oci.md
View file

@ -44,7 +44,11 @@ response = completion(
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
# Provide either the private key string OR the path to the key file:
# Option 1: pass the private key as a string
oci_key=<string_with_content_of_oci_key>,
# Option 2: pass the private key file path
# oci_key_file="<path/to/oci_key.pem>",
oci_compartment_id=<oci_compartment_id>,
)
print(response)
@ -67,7 +71,11 @@ response = completion(
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
# Provide either the private key string OR the path to the key file:
# Option 1: pass the private key as a string
oci_key=<string_with_content_of_oci_key>,
# Option 2: pass the private key file path
# oci_key_file="<path/to/oci_key.pem>",
oci_compartment_id=<oci_compartment_id>,
)
for chunk in response:

219
docs/my-website/docs/providers/vercel_ai_gateway.md Normal file
View file

@ -0,0 +1,219 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Vercel AI Gateway
## Overview
| Property | Details |
|-------|-------|
| Description | Vercel AI Gateway provides a unified interface to access multiple AI providers through a single endpoint, with built-in caching, rate limiting, and analytics. |
| Provider Route on LiteLLM | `vercel_ai_gateway/` |
| Link to Provider Doc | [Vercel AI Gateway Documentation ↗](https://vercel.com/docs/ai-gateway) |
| Base URL | `https://ai-gateway.vercel.sh/v1` |
| Supported Operations | `/chat/completions`, `/models` |
<br />
<br />
https://vercel.com/docs/ai-gateway
**We support ALL models available through Vercel AI Gateway, just set `vercel_ai_gateway/` as a prefix when sending completion requests**
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "" # your Vercel AI Gateway API key
# OR
os.environ["VERCEL_OIDC_TOKEN"] = "" # your Vercel OIDC token for authentication
```
## Optional Variables
```python showLineNumbers title="Environment Variables"
os.environ["VERCEL_SITE_URL"] = "" # your site url
# OR
os.environ["VERCEL_APP_NAME"] = "" # your app name
```
Note: see the [Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key) for instructions on obtaining a key.
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Vercel AI Gateway Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Vercel AI Gateway call
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Vercel AI Gateway Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Vercel AI Gateway call with streaming
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
## Usage - LiteLLM Proxy
Add the following to your LiteLLM Proxy configuration file:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4o-gateway
litellm_params:
model: vercel_ai_gateway/openai/gpt-4o
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
- model_name: claude-4-sonnet-gateway
litellm_params:
model: vercel_ai_gateway/anthropic/claude-4-sonnet
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
```
Start your LiteLLM Proxy server:
```bash showLineNumbers title="Start LiteLLM Proxy"
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
<Tabs>
<TabItem value="openai-sdk" label="OpenAI SDK">
```python showLineNumbers title="Vercel AI Gateway via Proxy - Non-streaming"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-proxy-api-key" # Your proxy API key
)
# Non-streaming response
response = client.chat.completions.create(
model="gpt-4o-gateway",
messages=[{"role": "user", "content": "Hello, how are you?"}]
)
print(response.choices[0].message.content)
```
```python showLineNumbers title="Vercel AI Gateway via Proxy - Streaming"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-proxy-api-key" # Your proxy API key
)
# Streaming response
response = client.chat.completions.create(
model="gpt-4o-gateway",
messages=[{"role": "user", "content": "Hello, how are you?"}],
stream=True
)
for chunk in response:
if chunk.choices[0].delta.content is not None:
print(chunk.choices[0].delta.content, end="")
```
</TabItem>
<TabItem value="litellm-sdk" label="LiteLLM SDK">
```python showLineNumbers title="Vercel AI Gateway via Proxy - LiteLLM SDK"
import litellm
# Configure LiteLLM to use your proxy
response = litellm.completion(
model="litellm_proxy/gpt-4o-gateway",
messages=[{"role": "user", "content": "Hello, how are you?"}],
api_base="http://localhost:4000",
api_key="your-proxy-api-key"
)
print(response.choices[0].message.content)
```
```python showLineNumbers title="Vercel AI Gateway via Proxy - LiteLLM SDK Streaming"
import litellm
# Configure LiteLLM to use your proxy with streaming
response = litellm.completion(
model="litellm_proxy/gpt-4o-gateway",
messages=[{"role": "user", "content": "Hello, how are you?"}],
api_base="http://localhost:4000",
api_key="your-proxy-api-key",
stream=True
)
for chunk in response:
if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
print(chunk.choices[0].delta.content, end="")
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Vercel AI Gateway via Proxy - cURL"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-d '{
"model": "gpt-4o-gateway",
"messages": [{"role": "user", "content": "Hello, how are you?"}]
}'
```
```bash showLineNumbers title="Vercel AI Gateway via Proxy - cURL Streaming"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-d '{
"model": "gpt-4o-gateway",
"messages": [{"role": "user", "content": "Hello, how are you?"}],
"stream": true
}'
```
</TabItem>
</Tabs>
For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
## Additional Resources
- [Vercel AI Gateway Documentation](https://vercel.com/docs/ai-gateway)

4
docs/my-website/docs/providers/vertex_image.md
View file

@ -18,7 +18,7 @@ import litellm
# Generate a single image
response = await litellm.aimage_generation(
prompt="An olympic size swimming pool with crystal clear water and modern architecture",
model="vertex_ai/imagen-4.0-generate-preview-06-06",
model="vertex_ai/imagen-4.0-generate-001",
vertex_ai_project="your-project-id",
vertex_ai_location="us-central1",
)
@ -34,7 +34,7 @@ print(response.data[0].url)
model_list:
- model_name: vertex-imagen
litellm_params:
model: vertex_ai/imagen-4.0-generate-preview-06-06
model: vertex_ai/imagen-4.0-generate-001
vertex_ai_project: "your-project-id"
vertex_ai_location: "us-central1"
vertex_ai_credentials: "path/to/service-account.json" # Optional if using environment auth

136
docs/my-website/docs/providers/vertex_partner.md
View file

@ -15,6 +15,7 @@ import TabItem from '@theme/TabItem';
| Mistral | `vertex_ai/mistral-*` | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) |
| AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) |
| Qwen | `vertex_ai/qwen/*` | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) |
| OpenAI (GPT-OSS) | `vertex_ai/openai/gpt-oss-*` | [Vertex AI - GPT-OSS Models](https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/) |
| Model Garden | `vertex_ai/openai/{MODEL_ID}` or `vertex_ai/{MODEL_ID}` | [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
## Vertex AI - Anthropic (Claude)
@ -658,6 +659,141 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
</Tabs>
## VertexAI GPT-OSS Models
| Property | Details |
|----------|---------|
| Provider Route | `vertex_ai/openai/{MODEL}` |
| Vertex Documentation | [Vertex AI - GPT-OSS Models](https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/) |
**LiteLLM Supports all Vertex AI GPT-OSS Models.** Ensure you use the `vertex_ai/openai/` prefix for all Vertex AI GPT-OSS models.
| Model Name | Usage |
|------------------|------------------------------|
| vertex_ai/openai/gpt-oss-20b-maas | `completion('vertex_ai/openai/gpt-oss-20b-maas', messages)` |
#### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
model = "openai/gpt-oss-20b-maas"
vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
response = completion(
model="vertex_ai/" + model,
messages=[{"role": "user", "content": "hi"}],
vertex_ai_project=vertex_ai_project,
vertex_ai_location=vertex_ai_location,
)
print("\nModel Response", response)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: gpt-oss
litellm_params:
model: vertex_ai/openai/gpt-oss-20b-maas
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-central1"
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-oss", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
#### Usage - `reasoning_effort`
GPT-OSS models support the `reasoning_effort` parameter for enhanced reasoning capabilities.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="vertex_ai/openai/gpt-oss-20b-maas",
messages=[{"role": "user", "content": "Solve this complex problem step by step"}],
reasoning_effort="low", # Options: "minimal", "low", "medium", "high"
vertex_ai_project="your-vertex-project",
vertex_ai_location="us-central1",
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-oss
litellm_params:
model: vertex_ai/openai/gpt-oss-20b-maas
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-central1"
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-d '{
"model": "gpt-oss",
"messages": [{"role": "user", "content": "Solve this complex problem step by step"}],
"reasoning_effort": "low"
}'
```
</TabItem>
</Tabs>
## Model Garden
:::tip

59
docs/my-website/docs/providers/volcano.md
View file

@ -3,7 +3,7 @@ https://www.volcengine.com/docs/82379/1263482
:::tip
**We support ALL Volcengine NIM models, just set `model=volcengine/<any-model-on-volcengine>` as a prefix when sending litellm requests**
**We support ALL Volcengine models including Chat and Embeddings, just set `model=volcengine/<any-model-on-volcengine>` as a prefix when sending litellm requests**
:::
@ -11,6 +11,8 @@ https://www.volcengine.com/docs/82379/1263482
```python
# env variable
os.environ['VOLCENGINE_API_KEY']
# or
os.environ['ARK_API_KEY']
```
## Sample Usage
@ -64,9 +66,42 @@ for chunk in response:
print(chunk)
```
## Sample Usage - Embedding
```python
from litellm import embedding
import os
## Supported Models - 💥 ALL Volcengine NIM Models Supported!
We support ALL `volcengine` models, just set `volcengine/<OUR_ENDPOINT_ID>` as a prefix when sending completion requests
os.environ['VOLCENGINE_API_KEY'] = ""
response = embedding(
model="volcengine/doubao-embedding-text-240715",
input=["hello world", "good morning"]
)
print(response)
```
### Supported Embedding Models
- `doubao-embedding-large` (2048 dimensions)
- `doubao-embedding-large-text-250515` (2048 dimensions)
- `doubao-embedding-large-text-240915` (4096 dimensions)
- `doubao-embedding` (2560 dimensions)
- `doubao-embedding-text-240715` (2560 dimensions)
### Embedding Parameters
```python
from litellm import embedding
response = embedding(
model="volcengine/doubao-embedding-text-240715",
input=["sample text"],
encoding_format="float", # optional: "float" (default), "base64"
user="user-123", # optional: user identifier for tracking
)
```
## Supported Models - 💥 ALL Volcengine Models Supported!
We support ALL `volcengine` models for both chat completions and embeddings:
- **Chat Models**: Set `volcengine/<OUR_ENDPOINT_ID>` as a prefix when sending completion requests
- **Embedding Models**: Use the specific model names listed above (e.g., `volcengine/doubao-embedding-text-240715`)
## Sample Usage - LiteLLM Proxy
@ -74,14 +109,21 @@ We support ALL `volcengine` models, just set `volcengine/<OUR_ENDPOINT_ID>` as a
```yaml
model_list:
# Chat model
- model_name: volcengine-model
litellm_params:
model: volcengine/<OUR_ENDPOINT_ID>
api_key: os.environ/VOLCENGINE_API_KEY
# Embedding model
- model_name: volcengine-embedding
litellm_params:
model: volcengine/doubao-embedding-text-240715
api_key: os.environ/VOLCENGINE_API_KEY
```
### Send Request
#### Chat Completion
```shell
curl --location 'http://localhost:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
@ -95,4 +137,15 @@ curl --location 'http://localhost:4000/chat/completions' \
}
]
}'
```
#### Embedding
```shell
curl --location 'http://localhost:4000/embeddings' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "volcengine-embedding",
"input": ["hello world", "good morning"]
}'
```

2
docs/my-website/docs/proxy/access_control.md
View file

@ -4,7 +4,7 @@ Role-based access control (RBAC) is based on Organizations, Teams and Internal U
- `Organizations` are the top-level entities that contain Teams.
- `Team` - A Team is a collection of multiple `Internal Users`
- `Internal Users` - users that can create keys, make LLM API calls, view usage on LiteLLM
- `Internal Users` - users that can create keys, make LLM API calls, view usage on LiteLLM. Users can be on multiple teams.
- `Roles` define the permissions of an `Internal User`
- `Virtual Keys` - Keys are used for authentication to the LiteLLM API. Keys are tied to a `Internal User` and `Team`

7
docs/my-website/docs/proxy/admin_ui_sso.md
View file

@ -235,6 +235,13 @@ Example setting a local image (on your container)
```shell
UI_LOGO_PATH="ui_images/logo.jpg"
```
#### Or set your logo directly from Admin UI:
<div style={{ display: 'flex', gap: '12px', alignItems: 'center' }}>
<Image img={require('../../img/admin_settings_ui_theme.png')} />
<Image img={require('../../img/admin_settings_ui_theme_logo.png')} />
</div>
#### Set Custom Color Theme
- Navigate to [/enterprise/enterprise_ui](https://github.com/BerriAI/litellm/blob/main/enterprise/enterprise_ui/_enterprise_colors.json)
- Inside the `enterprise_ui` directory, rename `_enterprise_colors.json` to `enterprise_colors.json`

4
docs/my-website/docs/proxy/call_hooks.md
View file

@ -6,6 +6,10 @@ import Image from '@theme/IdealImage';
- Reject data before making llm api calls / before returning the response
- Enforce 'user' param for all openai endpoint calls
:::tip
**Understanding Callback Hooks?** Check out our [Callback Management Guide](../observability/callback_management.md) to understand the differences between proxy-specific hooks like `async_pre_call_hook` and general logging hooks like `async_log_success_event`.
:::
See a complete example with our [parallel request rate limiter](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/parallel_request_limiter.py)
## Quick Start

32
docs/my-website/docs/proxy/config_settings.md
View file

@ -21,7 +21,7 @@ litellm_settings:
failure_callback: ["sentry"] # list of failure callbacks
callbacks: ["otel"] # list of callbacks - runs on success and failure
service_callbacks: ["datadog", "prometheus"] # logs redis, postgres failures on datadog, prometheus
turn_off_message_logging: boolean # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged.
turn_off_message_logging: boolean # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data.
redact_user_api_key_info: boolean # Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging.
langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] # default tags for Langfuse Logging
@ -131,7 +131,7 @@ general_settings:
| failure_callback | array of strings | List of failure callbacks [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
| callbacks | array of strings | List of callbacks - runs on success and failure [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
| service_callbacks | array of strings | System health monitoring - Logs redis, postgres failures on specified services (e.g. datadog, prometheus) [Doc Metrics](prometheus) |
| turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged [Proxy Logging](logging) |
| turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data [Proxy Logging](logging) |
| modify_params | boolean | If true, allows modifying the parameters of the request before it is sent to the LLM provider |
| enable_preview_features | boolean | If true, enables preview features - e.g. Azure O1 Models with streaming support.|
| redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) |
@ -236,7 +236,7 @@ Most values can also be set via `litellm_settings`. If you see overlapping value
```yaml
router_settings:
routing_strategy: usage-based-routing-v2 # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle"
routing_strategy: simple-shuffle # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle" - RECOMMENDED for best performance
redis_host: <your-redis-host> # string
redis_password: <your-redis-password> # string
redis_port: <your-redis-port> # string
@ -335,12 +335,19 @@ router_settings:
| ANTHROPIC_API_KEY | API key for Anthropic service
| ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
| AWS_ACCESS_KEY_ID | Access Key ID for AWS services
| AWS_BATCH_ROLE_ARN | ARN of the AWS IAM role for batch operations
| AWS_DEFAULT_REGION | Default AWS region for service interactions when AWS_REGION is not set
| AWS_PROFILE_NAME | AWS CLI profile name to be used
| AWS_REGION | AWS region for service interactions (takes precedence over AWS_DEFAULT_REGION)
| AWS_REGION_NAME | Default AWS region for service interactions
| AWS_ROLE_ARN | ARN of the AWS IAM role to assume for authentication
| AWS_ROLE_NAME | Role name for AWS IAM usage
| AWS_S3_BUCKET_NAME | Name of the AWS S3 bucket for file operations
| AWS_S3_OUTPUT_BUCKET_NAME | Name of the AWS S3 output bucket for batch operations
| AWS_SECRET_ACCESS_KEY | Secret Access Key for AWS services
| AWS_SESSION_NAME | Name for AWS session
| AWS_WEB_IDENTITY_TOKEN | Web identity token for AWS
| AWS_WEB_IDENTITY_TOKEN_FILE | Path to file containing web identity token for AWS
| AZURE_API_VERSION | Version of the Azure API being used
| AZURE_AUTHORITY_HOST | Azure authority host URL
| AZURE_CERTIFICATE_PASSWORD | Password for Azure OpenAI certificate
@ -376,6 +383,8 @@ router_settings:
| CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
| CLOUDZERO_API_KEY | CloudZero API key for authentication
| CLOUDZERO_CONNECTION_ID | CloudZero connection ID for data submission
| CLOUDZERO_EXPORT_INTERVAL_MINUTES | Interval in minutes for CloudZero data export operations
| CLOUDZERO_MAX_FETCHED_DATA_RECORDS | Maximum number of data records to fetch from CloudZero
| CLOUDZERO_TIMEZONE | Timezone for date handling (default: UTC)
| CONFIG_FILE_PATH | File path for configuration file
| CONFIDENT_API_KEY | API key for DeepEval integration
@ -408,6 +417,7 @@ router_settings:
| DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3
| DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096
| DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512
| DEFAULT_CLIENT_DISCONNECT_CHECK_TIMEOUT_SECONDS | Timeout in seconds for checking client disconnection. Default is 1
| DEFAULT_COOLDOWN_TIME_SECONDS | Duration in seconds to cooldown a model after failures. Default is 5
| DEFAULT_CRON_JOB_LOCK_TTL_SECONDS | Time-to-live for cron job locks in seconds. Default is 60 (1 minute)
| DEFAULT_FAILURE_THRESHOLD_PERCENT | Threshold percentage of failures to cool down a deployment. Default is 0.5 (50%)
@ -427,12 +437,17 @@ router_settings:
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available**
| DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7
| DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03
| DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0
| DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET | Default high reasoning effort thinking budget. Default is 4096
| DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET | Default low reasoning effort thinking budget. Default is 1024
| DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET | Default medium reasoning effort thinking budget. Default is 2048
| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET | Default minimal reasoning effort thinking budget. Default is 512
| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH | Default minimal reasoning effort thinking budget for Gemini 2.5 Flash. Default is 512
| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE | Default minimal reasoning effort thinking budget for Gemini 2.5 Flash Lite. Default is 512
| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO | Default minimal reasoning effort thinking budget for Gemini 2.5 Pro. Default is 512
| DEFAULT_REDIS_SYNC_INTERVAL | Default Redis synchronization interval in seconds. Default is 1
| DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND | Default price per second for Replicate GPU. Default is 0.001400
| DEFAULT_REPLICATE_POLLING_DELAY_SECONDS | Default delay in seconds for Replicate polling. Default is 1
@ -458,7 +473,6 @@ router_settings:
| EMAIL_SIGNATURE | Custom HTML footer/signature for all emails. Can include HTML tags for formatting and links.
| EMAIL_SUBJECT_INVITATION | Custom subject template for invitation emails.
| EMAIL_SUBJECT_KEY_CREATED | Custom subject template for key creation emails.
| EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False**
| FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4
| FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16
| FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56
@ -508,6 +522,8 @@ router_settings:
| GOOGLE_KMS_RESOURCE_NAME | Name of the resource in Google KMS
| GUARDRAILS_AI_API_BASE | Base URL for Guardrails AI API
| HEALTH_CHECK_TIMEOUT_SECONDS | Timeout in seconds for health checks. Default is 60
| HEROKU_API_BASE | Base URL for Heroku API
| HEROKU_API_KEY | API key for Heroku services
| HF_API_BASE | Base URL for Hugging Face API
| HCP_VAULT_ADDR | Address for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_CLIENT_CERT | Path to client certificate for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
@ -551,9 +567,11 @@ router_settings:
| LASSO_USER_ID | User ID for Lasso service
| LASSO_CONVERSATION_ID | Conversation ID for Lasso service
| LENGTH_OF_LITELLM_GENERATED_KEY | Length of keys generated by LiteLLM. Default is 16
| LEGACY_MULTI_INSTANCE_RATE_LIMITING | Flag to enable legacy multi-instance rate limiting. **Default is False**
| LITERAL_API_KEY | API key for Literal integration
| LITERAL_API_URL | API URL for Literal service
| LITERAL_BATCH_SIZE | Batch size for Literal operations
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
| LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
@ -566,6 +584,11 @@ router_settings:
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
| LITELLM_LOG | Enable detailed logging for LiteLLM
| LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file
| LITELLM_LOGGER_NAME | Name for OTEL logger
| LITELLM_METER_NAME | Name for OTEL Meter
| LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL
| LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL
| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
@ -576,6 +599,7 @@ router_settings:
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LOGFIRE_TOKEN | Token for Logfire logging service
| MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000
| MAX_STRING_LENGTH_PROMPT_IN_DB | Maximum length for strings in spend logs when sanitizing request bodies. Strings longer than this will be truncated. Default is 1000
| MAX_IN_MEMORY_QUEUE_FLUSH_COUNT | Maximum count for in-memory queue flush operations. Default is 1000
| MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the long side of high-resolution images. Default is 2000
| MAX_REDIS_BUFFER_DEQUEUE_COUNT | Maximum count for Redis buffer dequeue operations. Default is 100

304
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -17,7 +17,6 @@ LiteLLM automatically tracks spend for all known models. See our [model cost map
**Step2** Send `/chat/completions` request
<Tabs>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
```python
@ -505,11 +504,11 @@ litellm_settings:
### Disable user-agent tracking
You can disable user-agent tracking by setting `litellm_settings.disable_user_agent_tracking` to `true`.
You can disable user-agent tracking by setting `litellm_settings.disable_add_user_agent_to_request_tags` to `true`.
```yaml
litellm_settings:
disable_user_agent_tracking: true
disable_add_user_agent_to_request_tags: true
```
## ✨ (Enterprise) Generate Spend Reports
@ -860,6 +859,303 @@ Log specific key,value pairs as part of the metadata for a spend log
:::info
Logging specific key,value pairs in spend logs metadata is an enterprise feature. [See here](./enterprise.md#tracking-spend-with-custom-metadata)
Logging specific key,value pairs in spend logs metadata is an enterprise feature.
:::
Requirements:
- Virtual Keys & a database should be set up, see [virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys)
#### Usage - /chat/completions requests with special spend logs metadata
<Tabs>
<TabItem value="key" label="Set on Key">
```bash
curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {
"spend_logs_metadata": {
"hello": "world"
}
}
}
'
```
</TabItem>
<TabItem value="team" label="Set on Team">
```bash
curl -L -X POST 'http://0.0.0.0:4000/team/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {
"spend_logs_metadata": {
"hello": "world"
}
}
}
'
```
</TabItem>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
Set `extra_body={"metadata": { }}` to `metadata` you want to pass
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(
model="gpt-3.5-turbo",
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
extra_body={
"metadata": {
"spend_logs_metadata": {
"hello": "world"
}
}
}
)
print(response)
```
**Using Headers:**
```python
import openai
client = openai.OpenAI(
api_key="sk-1234",
base_url="http://0.0.0.0:4000"
)
# Pass spend logs metadata via headers
response = client.chat.completions.create(
model="gpt-3.5-turbo",
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
extra_headers={
"x-litellm-spend-logs-metadata": '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}'
}
)
print(response)
```
</TabItem>
<TabItem value="openai js" label="OpenAI JS">
```js
const openai = require('openai');
async function runOpenAI() {
const client = new openai.OpenAI({
apiKey: 'sk-1234',
baseURL: 'http://0.0.0.0:4000'
});
try {
const response = await client.chat.completions.create({
model: 'gpt-3.5-turbo',
messages: [
{
role: 'user',
content: "this is a test request, write a short poem"
},
],
metadata: {
spend_logs_metadata: { // 👈 Key Change
hello: "world"
}
}
});
console.log(response);
} catch (error) {
console.log("got this exception from server");
console.error(error);
}
}
// Call the asynchronous function
runOpenAI();
```
**Using Headers:**
```js
const openai = require('openai');
async function runOpenAI() {
const client = new openai.OpenAI({
apiKey: 'sk-1234',
baseURL: 'http://0.0.0.0:4000'
});
try {
const response = await client.chat.completions.create({
model: 'gpt-3.5-turbo',
messages: [
{
role: 'user',
content: "this is a test request, write a short poem"
},
]
}, {
headers: {
'x-litellm-spend-logs-metadata': '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}'
}
});
console.log(response);
} catch (error) {
console.log("got this exception from server");
console.error(error);
}
}
// Call the asynchronous function
runOpenAI();
```
</TabItem>
<TabItem value="Curl" label="Curl Request">
Pass `metadata` as part of the request body
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
"metadata": {
"spend_logs_metadata": {
"hello": "world"
}
}
}'
```
</TabItem>
<TabItem value="headers" label="Using Headers">
Pass `x-litellm-spend-logs-metadata` as a request header with JSON string
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--header 'x-litellm-spend-logs-metadata: {"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
<TabItem value="langchain" label="Langchain">
```python
from langchain.chat_models import ChatOpenAI
from langchain.prompts.chat import (
ChatPromptTemplate,
HumanMessagePromptTemplate,
SystemMessagePromptTemplate,
)
from langchain.schema import HumanMessage, SystemMessage
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:4000",
model = "gpt-3.5-turbo",
temperature=0.1,
extra_body={
"metadata": {
"spend_logs_metadata": {
"hello": "world"
}
}
}
)
messages = [
SystemMessage(
content="You are a helpful assistant that im using to make a test request to."
),
HumanMessage(
content="test from litellm. tell me why it's amazing in 1 sentence"
),
]
response = chat(messages)
print(response)
```
</TabItem>
</Tabs>
#### Viewing Spend w/ custom metadata
#### `/spend/logs` Request Format
```bash
curl -X GET "http://0.0.0.0:4000/spend/logs?request_id=<your-call-id" \ # e.g.: chatcmpl-9ZKMURhVYSi9D6r6PJ9vLcayIK0Vm
-H "Authorization: Bearer sk-1234"
```
#### `/spend/logs` Response Format
```bash
[
{
"request_id": "chatcmpl-9ZKMURhVYSi9D6r6PJ9vLcayIK0Vm",
"call_type": "acompletion",
"metadata": {
"user_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"user_api_key_alias": null,
"spend_logs_metadata": { # 👈 LOGGED CUSTOM METADATA
"hello": "world"
},
"user_api_key_team_id": null,
"user_api_key_user_id": "116544810872468347480",
"user_api_key_team_alias": null
},
}
]
```

13
docs/my-website/docs/proxy/deploy.md
View file

@ -12,10 +12,7 @@ To start using Litellm, run the following commands in a shell:
```bash
# Get the code
git clone https://github.com/BerriAI/litellm
# Go to folder
cd litellm
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
@ -1010,5 +1007,13 @@ User-agent: *
Disallow: /
```
## Deployment FAQ
**Q: Is Postgres the only supported database, or do you support other ones (like Mongo)?**
A: We explored MySQL but that was hard to maintain and led to bugs for customers. Currently, PostgreSQL is our primary supported database for production deployments.
**Q: If there is Postgres downtime, how does LiteLLM react? Does it fail-open or is there API downtime?**
A: You can gracefully handle DB unavailability if it's on your VPC. See our production guide for more details: [Gracefully Handle DB Unavailability](https://docs.litellm.ai/docs/proxy/prod#6-if-running-litellm-on-vpc-gracefully-handle-db-unavailability)

36
docs/my-website/docs/proxy/docker_quick_start.md
View file

@ -2,7 +2,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Getting Started - E2E Tutorial
# E2E Tutorial
End-to-End tutorial for LiteLLM Proxy to:
- Add an Azure OpenAI model
@ -13,7 +13,7 @@ End-to-End tutorial for LiteLLM Proxy to:
## Pre-Requisites
- Install LiteLLM Docker Image ** OR ** LiteLLM CLI (pip package)
- Install LiteLLM Docker Image **OR** LiteLLM CLI (pip package)
<Tabs>
@ -35,6 +35,30 @@ $ pip install 'litellm[proxy]'
</TabItem>
<TabItem value="docker-compose" label="Docker Compose (Proxy + DB)">
Use this docker compose to spin up the proxy with a postgres database running locally.
```bash
# Get the docker compose file
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# Add the litellm salt key - you cannot change this after adding a model
# It is used to encrypt / decrypt your LLM API Key credentials
# We recommend - https://1password.com/password-generator/
# password generator to get a random hash for litellm salt key
echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
source .env
# Start
docker-compose up
```
</TabItem>
</Tabs>
## 1. Add a model
@ -43,6 +67,8 @@ Control LiteLLM Proxy with a config.yaml file.
Setup your config.yaml with your azure model.
Note: When using the proxy with a database, you can also **just add models via UI** (UI is available on `/ui` route).
```yaml
model_list:
- model_name: gpt-4o
@ -252,15 +278,15 @@ See All General Settings [here](http://localhost:3000/docs/proxy/configs#all-set
- **Description**:
- Set a `master key`, this is your Proxy Admin key - you can use this to create other keys (🚨 must start with `sk-`).
- **Usage**:
- ** Set on config.yaml** set your master key under `general_settings:master_key`, example -
- **Set on config.yaml** set your master key under `general_settings:master_key`, example -
`master_key: sk-1234`
- ** Set env variable** set `LITELLM_MASTER_KEY`
- **Set env variable** set `LITELLM_MASTER_KEY`
2. **`database_url`** (str)
- **Description**:
- Set a `database_url`, this is the connection to your Postgres DB, which is used by litellm for generating keys, users, teams.
- **Usage**:
- ** Set on config.yaml** set your `database_url` under `general_settings:database_url`, example -
- **Set on config.yaml** set your `database_url` under `general_settings:database_url`, example -
`database_url: "postgresql://..."`
- Set `DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname>` in your env

214
docs/my-website/docs/proxy/enterprise.md
View file

@ -357,221 +357,13 @@ curl -X GET "http://0.0.0.0:4000/spend/tags" \
"total_spend": 0.000224
}
]
```
:::tip
For comprehensive spend tracking features including budgets, alerts, and detailed analytics, check out [Spend Tracking](https://docs.litellm.ai/docs/proxy/cost_tracking).
### Tracking Spend with custom metadata
:::
Requirements:
- Virtual Keys & a database should be set up, see [virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys)
#### Usage - /chat/completions requests with special spend logs metadata
<Tabs>
<TabItem value="key" label="Set on Key">
```bash
curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {
"spend_logs_metadata": {
"hello": "world"
}
}
}
'
```
</TabItem>
<TabItem value="team" label="Set on Team">
```bash
curl -L -X POST 'http://0.0.0.0:4000/team/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {
"spend_logs_metadata": {
"hello": "world"
}
}
}
'
```
</TabItem>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
Set `extra_body={"metadata": { }}` to `metadata` you want to pass
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(
model="gpt-3.5-turbo",
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
extra_body={
"metadata": {
"spend_logs_metadata": {
"hello": "world"
}
}
}
)
print(response)
```
</TabItem>
<TabItem value="openai js" label="OpenAI JS">
```js
const openai = require('openai');
async function runOpenAI() {
const client = new openai.OpenAI({
apiKey: 'sk-1234',
baseURL: 'http://0.0.0.0:4000'
});
try {
const response = await client.chat.completions.create({
model: 'gpt-3.5-turbo',
messages: [
{
role: 'user',
content: "this is a test request, write a short poem"
},
],
metadata: {
spend_logs_metadata: { // 👈 Key Change
hello: "world"
}
}
});
console.log(response);
} catch (error) {
console.log("got this exception from server");
console.error(error);
}
}
// Call the asynchronous function
runOpenAI();
```
</TabItem>
<TabItem value="Curl" label="Curl Request">
Pass `metadata` as part of the request body
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
"metadata": {
"spend_logs_metadata": {
"hello": "world"
}
}
}'
```
</TabItem>
<TabItem value="langchain" label="Langchain">
```python
from langchain.chat_models import ChatOpenAI
from langchain.prompts.chat import (
ChatPromptTemplate,
HumanMessagePromptTemplate,
SystemMessagePromptTemplate,
)
from langchain.schema import HumanMessage, SystemMessage
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:4000",
model = "gpt-3.5-turbo",
temperature=0.1,
extra_body={
"metadata": {
"spend_logs_metadata": {
"hello": "world"
}
}
}
)
messages = [
SystemMessage(
content="You are a helpful assistant that im using to make a test request to."
),
HumanMessage(
content="test from litellm. tell me why it's amazing in 1 sentence"
),
]
response = chat(messages)
print(response)
```
</TabItem>
</Tabs>
#### Viewing Spend w/ custom metadata
#### `/spend/logs` Request Format
```bash
curl -X GET "http://0.0.0.0:4000/spend/logs?request_id=<your-call-id" \ # e.g.: chatcmpl-9ZKMURhVYSi9D6r6PJ9vLcayIK0Vm
-H "Authorization: Bearer sk-1234"
```
#### `/spend/logs` Response Format
```bash
[
{
"request_id": "chatcmpl-9ZKMURhVYSi9D6r6PJ9vLcayIK0Vm",
"call_type": "acompletion",
"metadata": {
"user_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"user_api_key_alias": null,
"spend_logs_metadata": { # 👈 LOGGED CUSTOM METADATA
"hello": "world"
},
"user_api_key_team_id": null,
"user_api_key_user_id": "116544810872468347480",
"user_api_key_team_alias": null
},
}
]
```
## Guardrails - Secret Detection/Redaction
❓ Use this to REDACT API Keys, Secrets sent in requests to an LLM.

5
docs/my-website/docs/proxy/guardrails/custom_guardrail.md
View file

@ -23,15 +23,14 @@ A CustomGuardrail has 4 methods to enforce guardrails
Create a new file called `custom_guardrail.py` and add this code to it
```python
from typing import Any, Dict, List, Literal, Optional, Union
from typing import Any, AsyncGenerator, Literal, Optional, Union
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import ModelResponseStream
class myCustomGuardrail(CustomGuardrail):

299
docs/my-website/docs/proxy/guardrails/noma_security.md Normal file
View file

@ -0,0 +1,299 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Noma Security
Use [Noma Security](https://noma.security/) to protect your LLM applications with comprehensive AI content moderation and safety guardrails.
## Quick Start
### 1. Define Guardrails on your LiteLLM config.yaml
Define your guardrails under the `guardrails` section:
```yaml showLineNumbers title="litellm config.yaml"
model_list:
- model_name: gpt-4o-mini
litellm_params:
model: openai/gpt-4o-mini
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "noma-guard"
litellm_params:
guardrail: noma
mode: "during_call"
api_key: os.environ/NOMA_API_KEY
api_base: os.environ/NOMA_API_BASE
- guardrail_name: "noma-pre-guard"
litellm_params:
guardrail: noma
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
api_base: os.environ/NOMA_API_BASE
```
#### Supported values for `mode`
- `pre_call` Run **before** LLM call, on **input**
- `post_call` Run **after** LLM call, on **input & output**
- `during_call` Run **during** LLM call, on **input**. Same as `pre_call` but runs in parallel with the LLM call. Response not returned until guardrail check completes
### 2. Start LiteLLM Gateway
```shell
litellm --config config.yaml --detailed_debug
```
### 3. Test request
<Tabs>
<TabItem label="Unsuccessful call" value="not-allowed">
Expect this to fail since the request contains harmful content:
```shell showLineNumbers title="Curl Request"
curl -i http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o-mini",
"messages": [
{"role": "user", "content": "Tell me how to hack into someone's email account"}
]
}'
```
Expected response on failure:
```json
{
"error": {
"message": "{\n \"error\": \"Request blocked by Noma guardrail\",\n \"details\": {\n \"prompt\": {\n \"harmfulContent\": {\n \"result\": true,\n \"confidence\": 0.95\n }\n }\n }\n }",
"type": "None",
"param": "None",
"code": "400"
}
}
```
</TabItem>
<TabItem label="Successful Call" value="allowed">
```shell showLineNumbers title="Curl Request"
curl -i http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o-mini",
"messages": [
{"role": "user", "content": "What is the capital of France?"}
]
}'
```
Expected response:
```json
{
"id": "chatcmpl-123",
"object": "chat.completion",
"created": 1677652288,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "The capital of France is Paris."
},
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 9,
"completion_tokens": 12,
"total_tokens": 21
}
}
```
</TabItem>
</Tabs>
## Supported Params
```yaml
guardrails:
- guardrail_name: "noma-guard"
litellm_params:
guardrail: noma
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
api_base: os.environ/NOMA_API_BASE
### OPTIONAL ###
# application_id: "my-app"
# monitor_mode: false
# block_failures: true
```
### Required Parameters
- **`api_key`**: Your Noma Security API key (set as `os.environ/NOMA_API_KEY` in YAML config)
### Optional Parameters
- **`api_base`**: Noma API base URL (defaults to `https://api.noma.security/`)
- **`application_id`**: Your application identifier (defaults to `"litellm"`)
- **`monitor_mode`**: If `true`, logs violations without blocking (defaults to `false`)
- **`block_failures`**: If `true`, blocks requests when guardrail API failures occur (defaults to `true`)
## Environment Variables
You can set these environment variables instead of hardcoding values in your config:
```shell
export NOMA_API_KEY="your-api-key-here"
export NOMA_API_BASE="https://api.noma.security/" # Optional
export NOMA_APPLICATION_ID="my-app" # Optional
export NOMA_MONITOR_MODE="false" # Optional
export NOMA_BLOCK_FAILURES="true" # Optional
```
## Advanced Configuration
### Monitor Mode
Use monitor mode to test your guardrails without blocking requests:
```yaml
guardrails:
- guardrail_name: "noma-monitor"
litellm_params:
guardrail: noma
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
monitor_mode: true # Log violations but don't block
```
### Handling API Failures
Control behavior when the Noma API is unavailable:
```yaml
guardrails:
- guardrail_name: "noma-failopen"
litellm_params:
guardrail: noma
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
block_failures: false # Allow requests to proceed if guardrail API fails
```
### Multiple Guardrails
Apply different configurations for input and output:
```yaml
guardrails:
- guardrail_name: "noma-strict-input"
litellm_params:
guardrail: noma
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
block_failures: true
- guardrail_name: "noma-monitor-output"
litellm_params:
guardrail: noma
mode: "post_call"
api_key: os.environ/NOMA_API_KEY
monitor_mode: true
```
## ✨ Pass Additional Parameters
Use `extra_body` to pass additional parameters to the Noma Security API call, such as dynamically setting the application ID for specific requests.
<Tabs>
<TabItem value="openai" label="OpenAI Python">
```python
import openai
client = openai.OpenAI(
api_key="your-api-key",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Hello, how are you?"}],
extra_body={
"guardrails": {
"noma-guard": {
"extra_body": {
"application_id": "my-specific-app-id"
}
}
}
}
)
```
</TabItem>
<TabItem value="curl" label="Curl">
```shell
curl 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o-mini",
"messages": [
{
"role": "user",
"content": "Hello, how are you?"
}
],
"guardrails": {
"noma-guard": {
"extra_body": {
"application_id": "my-specific-app-id"
}
}
}
}'
```
</TabItem>
</Tabs>
This allows you to override the default `application_id` parameter for specific requests, which is useful for tracking usage across different applications or components.
## Response Details
When content is blocked, Noma provides detailed information about the violations as JSON inside the `message` field, with the following structure:
```json
{
"error": "Request blocked by Noma guardrail",
"details": {
"prompt": {
"harmfulContent": {
"result": true,
"confidence": 0.95
},
"sensitiveData": {
"email": {
"result": true,
"entities": ["user@example.com"]
}
},
"bannedTopics": {
"violence": {
"result": true,
"confidence": 0.88
}
}
}
}
}
```

3
docs/my-website/docs/proxy/health.md
View file

@ -128,8 +128,11 @@ model_list:
api_key: "os.environ/OPENAI_API_KEY"
model_info:
mode: audio_speech
health_check_voice: alloy
```
You can specify a `health_check_voice` if you need to use a voice other than "alloy".
### Rerank Models
To run rerank health checks, specify the mode as "rerank" in your config for the relevant model.

68
docs/my-website/docs/proxy/load_balancing.md
View file

@ -13,6 +13,23 @@ For more details on routing strategies / params, see [Routing](../routing.md)
:::
## How Load Balancing Works
LiteLLM automatically distributes requests across multiple deployments of the same model using its built-in router. the proxy routes traffic to optimize performance and reliability.
"simple-shuffle" routing strategy is used by default
### Routing Strategies
| Strategy | Description | When to Use |
|----------|-------------|-------------|
| **simple-shuffle** (recommended) | Randomly distributes requests | General purpose, good for even load distribution |
| **least-busy** | Routes to deployment with fewest active requests | High concurrency scenarios |
| **usage-based-routing** (bad for perf) | Routes to deployment with lowest current usage (RPM/TPM) | When you want to respect rate limits evenly |
| **latency-based-routing** | Routes to fastest responding deployment | Latency-critical applications |
| **cost-based-routing** | Routes to deployment with lowest cost | Cost-sensitive applications |
## Quick Start - Load Balancing
#### Step 1 - Set deployments on config
@ -106,49 +123,14 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
]
}'
```
</TabItem>
<TabItem value="langchain" label="Langchain">
```python
from langchain.chat_models import ChatOpenAI
from langchain.prompts.chat import (
ChatPromptTemplate,
HumanMessagePromptTemplate,
SystemMessagePromptTemplate,
)
from langchain.schema import HumanMessage, SystemMessage
import os
os.environ["OPENAI_API_KEY"] = "anything"
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:4000",
model="gpt-3.5-turbo",
)
messages = [
SystemMessage(
content="You are a helpful assistant that im using to make a test request to."
),
HumanMessage(
content="test from litellm. tell me why it's amazing in 1 sentence"
),
]
response = chat(messages)
print(response)
```
</TabItem>
</Tabs>
### Test - Loadbalancing
In this request, the following will occur:
1. A rate limit exception will be raised
2. LiteLLM proxy will retry the request on the model group (default is 3).
2. LiteLLM proxy will retry the request on the model group (default retries are 3).
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
@ -256,4 +238,16 @@ model_group_alias: Optional[Dict[str, Union[str, RouterModelGroupAliasItem]]] =
class RouterModelGroupAliasItem(TypedDict):
model: str
hidden: bool # if 'True', don't return on `/v1/models`, `/v1/model/info`, `/v1/model_group/info`
```
```
### When You'll See Load Balancing in Action
**Immediate Effects:**
- Different deployments serve subsequent requests (visible in logs)
- Better response times during high traffic
**Observable Benefits:**
- **Higher throughput**: More requests handled simultaneously across deployments
- **Improved reliability**: If one deployment fails, traffic automatically routes to healthy ones
- **Better resource utilization**: Load spread evenly across all available deployments

2
docs/my-website/docs/proxy/logging.md
View file

@ -60,7 +60,7 @@ components in your system, including in logging tools.
### Redact Messages, Response Content
Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to your logging provider, but request metadata - e.g. spend, will still be tracked.
Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to your logging provider, but request metadata - e.g. spend, will still be tracked. Useful for privacy/compliance when handling sensitive data.
<Tabs>

5
docs/my-website/docs/proxy/prod.md
View file

@ -90,7 +90,7 @@ Recommended to do this for prod:
```yaml
router_settings:
routing_strategy: usage-based-routing-v2
routing_strategy: simple-shuffle # (default) - recommended for best performance
# redis_url: "os.environ/REDIS_URL"
redis_host: os.environ/REDIS_HOST
redis_port: os.environ/REDIS_PORT
@ -105,6 +105,9 @@ litellm_settings:
password: os.environ/REDIS_PASSWORD
```
> **WARNING**
**Usage-based routing is not recommended for production due to performance impacts.** Use `simple-shuffle` (default) for optimal performance in high-traffic scenarios.
## 5. Disable 'load_dotenv'
Set `export LITELLM_MODE="PRODUCTION"`

25
docs/my-website/docs/proxy/prometheus.md
View file

@ -63,7 +63,7 @@ Use this for for tracking per [user, key, team, etc.](virtual_keys)
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_spend_metric` | Total Spend, per `"user", "key", "model", "team", "end-user"` |
| `litellm_spend_metric` | Total Spend, per `"end_user", "hashed_api_key", "api_key_alias", "model", "team", "team_alias", "user"` |
| `litellm_total_tokens_metric` | input + output tokens per `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model"` |
| `litellm_input_tokens_metric` | input tokens per `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model"` |
| `litellm_output_tokens_metric` | output tokens per `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model"` |
@ -73,9 +73,9 @@ Use this for for tracking per [user, key, team, etc.](virtual_keys)
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_team_max_budget_metric` | Max Budget for Team Labels: `"team_id", "team_alias"`|
| `litellm_remaining_team_budget_metric` | Remaining Budget for Team (A team created on LiteLLM) Labels: `"team_id", "team_alias"`|
| `litellm_team_budget_remaining_hours_metric` | Hours before the team budget is reset Labels: `"team_id", "team_alias"`|
| `litellm_team_max_budget_metric` | Max Budget for Team Labels: `"team", "team_alias"`|
| `litellm_remaining_team_budget_metric` | Remaining Budget for Team (A team created on LiteLLM) Labels: `"team", "team_alias"`|
| `litellm_team_budget_remaining_hours_metric` | Hours before the team budget is reset Labels: `"team", "team_alias"`|
### Virtual Key - Budget
@ -119,8 +119,8 @@ Use this to track overall LiteLLM Proxy usage.
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "exception_status", "exception_class"` |
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code"` |
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "exception_status", "exception_class", "route"` |
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route"` |
## LLM Provider Metrics
@ -155,7 +155,7 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_remaining_requests_metric` | Track `x-ratelimit-remaining-requests` returned from LLM API Deployment. Labels: `"model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias"` |
| `litellm_remaining_tokens` | Track `x-ratelimit-remaining-tokens` return from LLM API Deployment. Labels: `"model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias"` |
| `litellm_remaining_tokens_metric` | Track `x-ratelimit-remaining-tokens` return from LLM API Deployment. Labels: `"model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias"` |
### Deployment State
| Metric Name | Description |
@ -167,16 +167,22 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_deployment_cooled_down` | Number of times a deployment has been cooled down by LiteLLM load balancing logic. Labels: `"litellm_model_name", "model_id", "api_base", "api_provider", "exception_status"` |
| `litellm_deployment_cooled_down` | Number of times a deployment has been cooled down by LiteLLM load balancing logic. Labels: `"litellm_model_name", "model_id", "api_base", "api_provider"` |
| `litellm_deployment_successful_fallbacks` | Number of successful fallback requests from primary model -> fallback model. Labels: `"requested_model", "fallback_model", "hashed_api_key", "api_key_alias", "team", "team_alias", "exception_status", "exception_class"` |
| `litellm_deployment_failed_fallbacks` | Number of failed fallback requests from primary model -> fallback model. Labels: `"requested_model", "fallback_model", "hashed_api_key", "api_key_alias", "team", "team_alias", "exception_status", "exception_class"` |
## Request Counting Metrics
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_requests_metric` | Total number of requests tracked per endpoint. Labels: `"end_user", "hashed_api_key", "api_key_alias", "model", "team", "team_alias", "user", "user_email"` |
## Request Latency Metrics
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model" |
| `litellm_overhead_latency_metric` | Latency overhead (seconds) added by LiteLLM processing - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model" |
| `litellm_overhead_latency_metric` | Latency overhead (seconds) added by LiteLLM processing - tracked for labels "model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias" |
| `litellm_llm_api_latency_metric` | Latency (seconds) for just the LLM API call - tracked for labels "model", "hashed_api_key", "api_key_alias", "team", "team_alias", "requested_model", "end_user", "user" |
| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias` [Note: only emitted for streaming requests] |
@ -486,7 +492,6 @@ Here is a screenshot of the metrics you can monitor with the LiteLLM Grafana Das
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_llm_api_failed_requests_metric` | **deprecated** use `litellm_proxy_failed_requests_metric` |
| `litellm_requests_metric` | **deprecated** use `litellm_proxy_total_requests_metric` |

5
docs/my-website/docs/proxy/quick_start.md
View file

@ -2,8 +2,9 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Quick Start
Quick start CLI, Config, Docker
# CLI - Quick Start
Setup LiteLLM Proxy quickly via CLI.
LiteLLM Server (LLM Gateway) manages:

4
docs/my-website/docs/proxy/request_headers.md
View file

@ -6,12 +6,16 @@ Special headers that are supported by LiteLLM.
`x-litellm-timeout` Optional[float]: The timeout for the request in seconds.
`x-litellm-stream-timeout` Optional[float]: The timeout for getting the first chunk of the response in seconds (only applies for streaming requests). [Demo Video](https://www.loom.com/share/8da67e4845ce431a98c901d4e45db0e5)
`x-litellm-enable-message-redaction`: Optional[bool]: Don't log the message content to logging integrations. Just track spend. [Learn More](./logging#redact-messages-response-content)
`x-litellm-tags`: Optional[str]: A comma separated list (e.g. `tag1,tag2,tag3`) of tags to use for [tag-based routing](./tag_routing) **OR** [spend-tracking](./enterprise.md#tracking-spend-for-custom-tags).
`x-litellm-num-retries`: Optional[int]: The number of retries for the request.
`x-litellm-spend-logs-metadata`: Optional[str]: JSON string containing custom metadata to include in spend logs. Example: `{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}`. [Learn More](../proxy/enterprise#tracking-spend-with-custom-metadata)
## Anthropic Headers
`anthropic-version` Optional[str]: The version of the Anthropic API to use.

31
docs/my-website/docs/proxy/self_serve.md
View file

@ -309,6 +309,37 @@ curl -X POST '<PROXY_BASE_URL>/team/new' \
</TabItem>
</Tabs>
### Team Member Rate Limits
Set a default tpm/rpm limit for an individual team member.
You can do this when creating a new team, or by updating an existing team.
<Tabs>
<TabItem value="ui" label="UI">
<Image img={require('../../img/create_team_member_rate_limits.png')} style={{ width: '600px', height: 'auto' }} />
</TabItem>
<TabItem value="api" label="API">
```bash
curl -X POST '<PROXY_BASE_URL>/team/new' \
-H 'Authorization: Bearer <PROXY_MASTER_KEY>' \
-H 'Content-Type: application/json' \
-D '{
"team_alias": "team_1",
"team_member_rpm_limit": 100,
"team_member_tpm_limit": 1000
}'
```
</TabItem>
</Tabs>
### Set default params for new teams
When you connect litellm to your SSO provider, litellm can auto-create teams. Use this to set the default `models`, `max_budget`, `budget_duration` for these auto-created teams.

9
docs/my-website/docs/proxy/tag_routing.md
View file

@ -5,6 +5,12 @@ This is useful for
- Implementing free / paid tiers for users
- Controlling model access per team, example Team A can access gpt-4 deployment A, Team B can access gpt-4 deployment B (LLM Access Control For Teams )
:::info
## See here for spend tags
- [Track spend per tag](cost_tracking#-custom-tags)
- [Setup Budgets per Virtual Key, Team](users)
:::
## Quick Start
### 1. Define tags on config.yaml
@ -324,7 +330,4 @@ Here's how to set up and use team-based tag routing using curl commands:
By following these steps and using these curl commands, you can implement and test team-based tag routing in your LiteLLM Proxy setup, ensuring that different teams are routed to the appropriate models or deployments based on their assigned tags.
## Other Tag Based Features
- [Track spend per tag](cost_tracking#-custom-tags)
- [Setup Budgets per Virtual Key, Team](users)

6
docs/my-website/docs/proxy/team_budgets.md
View file

@ -4,6 +4,12 @@ import TabItem from '@theme/TabItem';
# Setting Team Budgets
# Pre-Requisites
- You must set up a Postgres database (e.g. Supabase, Neon, etc.)
- To enable team member rate limits, set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` **before starting the proxy server**. Without this, team member rate limits will not be enforced.
Track spend, set budgets for your Internal Team
## Setting Monthly Team Budgets

10
docs/my-website/docs/proxy/timeout.md
View file

@ -38,9 +38,15 @@ $ litellm --config /path/to/config.yaml
</TabItem>
</Tabs>
### Custom Timeouts, Stream Timeouts - Per Model
For each model you can set `timeout` & `stream_timeout` under `litellm_params`
### Custom Timeouts & Stream Timeouts (Per Model)
For each model, you can set `timeout` and `stream_timeout` under `litellm_params`:
- **`timeout`** → maximum time for the *complete response*.
Use this to cap long-running completions.
- **`stream_timeout`** → maximum time to wait for the *first chunk* (i.e., first token) in a streaming response.
Use this to abort “hanging” providers (e.g., Bedrock slow start) and retry another model.
<Tabs>
<TabItem value="sdk" label="SDK">

100
docs/my-website/docs/proxy/user_keys.md
View file

@ -357,6 +357,106 @@ assert user.age == 25
</TabItem>
</Tabs>
## Using Tags for Categorization and Tracking
Tags allow you to categorize, filter, and track your LLM requests. Add tags to your metadata for better organization and analytics.
<Tabs>
<TabItem value="openai-python" label="OpenAI Python">
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello!"}],
extra_body={
"metadata": {
"tags": ["production", "customer-support", "urgent"],
"generation_name": "support-bot",
"trace_user_id": "user-123"
}
}
)
```
</TabItem>
<TabItem value="langchain-python" label="LangChain Python">
```python
from langchain_openai import ChatOpenAI
from langchain_core.messages import HumanMessage
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:4000",
model="gpt-4o",
extra_body={
"metadata": {
"tags": ["langchain-integration", "content-gen"],
"trace_user_id": "user-456"
}
}
)
response = chat.invoke([HumanMessage(content="Generate a blog post")])
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello!"}],
"metadata": {
"tags": ["api-test", "development"],
"trace_user_id": "test-user"
}
}'
```
</TabItem>
<TabItem value="openai-js" label="OpenAI JS">
```js
const { OpenAI } = require('openai');
const openai = new OpenAI({
apiKey: "sk-1234",
baseURL: "http://0.0.0.0:4000"
});
async function main() {
const response = await openai.chat.completions.create({
messages: [{ role: 'user', content: 'Hello!' }],
model: 'gpt-3.5-turbo',
metadata: {
tags: ["javascript-client", "api-test"],
trace_user_id: "js-user-789"
}
});
}
```
</TabItem>
</Tabs>
### Tag Benefits
- **Cost Tracking**: Monitor spending by project/team/feature
- **Analytics**: Filter requests by tags in logs and dashboards
- **Routing**: Use tags for conditional model routing
- **Debugging**: Easier troubleshooting with categorized requests
### Response Format
```json

2
docs/my-website/docs/proxy/user_management_heirarchy.md
View file

@ -9,5 +9,5 @@ LiteLLM supports a hierarchy of users, teams, organizations, and budgets.
- Organizations can have multiple teams. [API Reference](https://litellm-api.up.railway.app/#/organization%20management)
- Teams can have multiple users. [API Reference](https://litellm-api.up.railway.app/#/team%20management)
- Users can have multiple keys. [API Reference](https://litellm-api.up.railway.app/#/budget%20management)
- Users can have multiple keys, and be on multiple teams. [API Reference](https://litellm-api.up.railway.app/#/budget%20management)
- Keys can belong to either a team or a user. [API Reference](https://litellm-api.up.railway.app/#/end-user%20management)

8
docs/my-website/docs/proxy/users.md
View file

@ -58,6 +58,9 @@ You can:
**Step-by step tutorial on setting, resetting budgets on Teams here (API or using Admin UI)**
> **Prerequisite:**
> To enable team member rate limits, you must set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` before starting the proxy server. Without this, team member rate limits will not be enforced.
👉 [https://docs.litellm.ai/docs/proxy/team_budgets](https://docs.litellm.ai/docs/proxy/team_budgets)
:::
@ -793,6 +796,11 @@ Expected Response:
Enable multi-instance rate limiting with the env var `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"`
**Important Notes:**
- Setting `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"` is required for team member rate limits to function, not just for multi-instance scenarios.
- **Rate limits do not apply to proxy admin users.**
- When testing rate limits, use internal user roles (non-admin) to ensure limits are enforced as expected.
Changes:
- This moves to using async_increment instead of async_set_cache when updating current requests/tokens.
- The in-memory cache is synced with redis every 0.01s, to avoid calling redis for every request.

163
docs/my-website/docs/proxy/veo_video_generation.md Normal file
View file

@ -0,0 +1,163 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Veo Video Generation with Google AI Studio
Generate videos using Google's Veo model through LiteLLM's pass-through endpoints.
## Quick Start
LiteLLM allows you to use Google AI Studio's Veo video generation API through pass-through routes with zero configuration.
### 1. Add Google AI Studio API Key to your environment
```bash
export GEMINI_API_KEY="your_google_ai_studio_api_key"
```
### 2. Start LiteLLM Proxy
```bash
litellm
# RUNNING on http://0.0.0.0:4000
```
### 3. Generate Video
<Tabs>
<TabItem value="python" label="Python">
```python
import requests
import time
import json
# Configuration
BASE_URL = "http://localhost:4000/gemini/v1beta"
API_KEY = "anything" # Use "anything" as the key
headers = {
"x-goog-api-key": API_KEY,
"Content-Type": "application/json"
}
# Step 1: Initiate video generation
def generate_video(prompt):
url = f"{BASE_URL}/models/veo-3.0-generate-preview:predictLongRunning"
payload = {
"instances": [{
"prompt": prompt
}]
}
response = requests.post(url, headers=headers, json=payload)
response.raise_for_status()
data = response.json()
return data.get("name") # Operation name
# Step 2: Poll for completion
def wait_for_completion(operation_name):
operation_url = f"{BASE_URL}/{operation_name}"
while True:
response = requests.get(operation_url, headers=headers)
response.raise_for_status()
data = response.json()
if data.get("done", False):
# Extract video URI
video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
return video_uri
time.sleep(10) # Wait 10 seconds before next poll
# Step 3: Download video
def download_video(video_uri, filename="generated_video.mp4"):
# Replace Google URL with LiteLLM proxy URL
litellm_url = video_uri.replace(
"https://generativelanguage.googleapis.com/v1beta",
BASE_URL
)
response = requests.get(litellm_url, headers=headers, stream=True)
response.raise_for_status()
with open(filename, 'wb') as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
return filename
# Complete workflow
prompt = "A cat playing with a ball of yarn in a sunny garden"
print("Generating video...")
operation_name = generate_video(prompt)
print("Waiting for completion...")
video_uri = wait_for_completion(operation_name)
print("Downloading video...")
filename = download_video(video_uri)
print(f"Video saved as: {filename}")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash
# Step 1: Initiate video generation
curl -X POST "http://localhost:4000/gemini/v1beta/models/veo-3.0-generate-preview:predictLongRunning" \
-H "x-goog-api-key: anything" \
-H "Content-Type: application/json" \
-d '{
"instances": [{
"prompt": "A cat playing with a ball of yarn in a sunny garden"
}]
}'
# Response will include operation name:
# {"name": "operations/generate_12345"}
# Step 2: Poll for completion
curl -X GET "http://localhost:4000/gemini/v1beta/operations/generate_12345" \
-H "x-goog-api-key: anything"
# Step 3: Download video (when done=true)
curl -X GET "http://localhost:4000/gemini/v1beta/files/VIDEO_ID:download?alt=media" \
-H "x-goog-api-key: anything" \
--output generated_video.mp4
```
</TabItem>
</Tabs>
## Complete Example
For a full working example with error handling and logging, see our [Veo Video Generation Cookbook](https://github.com/BerriAI/litellm/blob/main/cookbook/veo_video_generation.py).
## How It Works
1. **Video Generation Request**: Send a prompt to Veo's `predictLongRunning` endpoint
2. **Operation Polling**: Monitor the long-running operation until completion
3. **File Download**: Download the generated video through LiteLLM's pass-through with automatic redirect handling
LiteLLM handles:
- ✅ Authentication with Google AI Studio
- ✅ Request routing and proxying
- ✅ Automatic redirect handling for file downloads
## Configuration Options
### Environment Variables
```bash
export GEMINI_API_KEY="your_google_ai_studio_api_key"
```

3
docs/my-website/docs/reasoning_content.md
View file

@ -12,7 +12,7 @@ Requires LiteLLM v1.63.0+
Supported Providers:
- Deepseek (`deepseek/`)
- Anthropic API (`anthropic/`)
- Bedrock (Anthropic + Deepseek) (`bedrock/`)
- Bedrock (Anthropic + Deepseek + GPT-OSS) (`bedrock/`)
- Vertex AI (Anthropic) (`vertexai/`)
- OpenRouter (`openrouter/`)
- XAI (`xai/`)
@ -20,6 +20,7 @@ Supported Providers:
- Vertex AI (`vertex_ai/`)
- Perplexity (`perplexity/`)
- Mistral AI (Magistral models) (`mistral/`)
- Groq (`groq/`)
LiteLLM will standardize the `reasoning_content` in the response and `thinking_blocks` in the assistant message.

3
docs/my-website/docs/rerank.md
View file

@ -118,4 +118,5 @@ curl http://0.0.0.0:4000/rerank \
| AWS Bedrock| [Usage](../docs/providers/bedrock#rerank-api) |
| HuggingFace| [Usage](../docs/providers/huggingface_rerank) |
| Infinity| [Usage](../docs/providers/infinity) |
| vLLM| [Usage](../docs/providers/vllm#rerank-endpoint) |
| vLLM| [Usage](../docs/providers/vllm#rerank-endpoint) |
| DeepInfra| [Usage](../docs/providers/deepinfra#rerank-endpoint) |

7
docs/my-website/docs/response_api.md
View file

@ -805,16 +805,17 @@ LiteLLM Proxy supports session management for non-OpenAI models. This allows you
Set `store_prompts_in_cold_storage: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the s3 bucket you specify.
```yaml
```yaml showLineNumbers title="config.yaml with Session Continuity"
litellm_settings:
callbacks: ["s3_v2"]
cold_storage_custom_logger: s3_v2
s3_callback_params: # learn more https://docs.litellm.ai/docs/proxy/logging#s3-buckets
s3_bucket_name: litellm-logs # AWS Bucket Name for S3
s3_region_name: us-west-2
s3_region_name: us-west-2
general_settings:
cold_storage_custom_logger: s3_v2
store_prompts_in_cold_storage: true
store_prompts_in_spend_logs: true
```
2. Make request 1 with no `previous_response_id` (new session)

284
docs/my-website/docs/routing.md
View file

@ -154,11 +154,153 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
## Advanced - Routing Strategies ⭐️
#### Routing Strategies - Weighted Pick, Rate Limit Aware, Least Busy, Latency Based, Cost Based
Router provides 4 strategies for routing your calls across multiple deployments:
Router provides multiple strategies for routing your calls across multiple deployments. **We recommend using `simple-shuffle` (default) for best performance in production.**
<Tabs>
<TabItem value="simple-shuffle" label="(Default) Weighted Pick - RECOMMENDED">
**Default and Recommended for Production** - Best performance with minimal latency overhead.
Picks a deployment based on the provided **Requests per minute (rpm) or Tokens per minute (tpm)**
If `rpm` or `tpm` is not provided, it randomly picks a deployment
You can also set a `weight` param, to specify which model should get picked when.
<Tabs>
<TabItem value="rpm" label="RPM-based shuffling">
##### **LiteLLM Proxy Config.yaml**
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-v-2
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
rpm: 900
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-functioncalling
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
rpm: 10
```
##### **Python SDK**
```python
from litellm import Router
import asyncio
model_list = [{ # list of model deployments
"model_name": "gpt-3.5-turbo", # model alias
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-2", # actual model name
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"rpm": 900, # requests per minute for this API
}
}, {
"model_name": "gpt-3.5-turbo",
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-functioncalling",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"rpm": 10,
}
},]
# init router
router = Router(model_list=model_list, routing_strategy="simple-shuffle")
async def router_acompletion():
response = await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}]
)
print(response)
return response
asyncio.run(router_acompletion())
```
</TabItem>
<TabItem value="weight" label="Weight-based shuffling">
##### **LiteLLM Proxy Config.yaml**
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-v-2
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
weight: 9
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-functioncalling
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
weight: 1
```
##### **Python SDK**
```python
from litellm import Router
import asyncio
model_list = [{
"model_name": "gpt-3.5-turbo", # model alias
"litellm_params": {
"model": "azure/chatgpt-v-2", # actual model name
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"weight": 9, # pick this 90% of the time
}
}, {
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/chatgpt-functioncalling",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"weight": 1,
}
}]
# init router
router = Router(model_list=model_list, routing_strategy="simple-shuffle")
async def router_acompletion():
response = await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}]
)
print(response)
return response
asyncio.run(router_acompletion())
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="usage-based-v2" label="Rate-Limit Aware v2 (ASYNC)">
> [!WARNING]
**Usage-based routing is not recommended for production due to performance impacts.** Use `simple-shuffle` (default) for optimal performance in high-traffic scenarios. Usage-based routing adds significant latency due to Redis operations for tracking usage across deployments.
**🎉 NEW** This is an async implementation of usage-based-routing.
**Filters out deployment if tpm/rpm limit exceeded** - If you pass in the deployment's tpm/rpm limits.
@ -209,7 +351,7 @@ router = Router(model_list=model_list,
redis_host=os.environ["REDIS_HOST"],
redis_password=os.environ["REDIS_PASSWORD"],
redis_port=os.environ["REDIS_PORT"],
routing_strategy="usage-based-routing-v2" # 👈 KEY CHANGE
routing_strategy="simple-shuffle" # 👈 RECOMMENDED - best performance
enable_pre_call_checks=True, # enables router rate limits for concurrent calls
)
@ -241,7 +383,7 @@ model_list:
rpm: 1000
router_settings:
routing_strategy: usage-based-routing-v2 # 👈 KEY CHANGE
routing_strategy: simple-shuffle # 👈 RECOMMENDED - best performance
redis_host: <your-redis-host>
redis_password: <your-redis-password>
redis_port: <your-redis-port>
@ -365,143 +507,7 @@ router_settings:
```
</TabItem>
<TabItem value="simple-shuffle" label="(Default) Weighted Pick (Async)">
**Default** Picks a deployment based on the provided **Requests per minute (rpm) or Tokens per minute (tpm)**
If `rpm` or `tpm` is not provided, it randomly picks a deployment
You can also set a `weight` param, to specify which model should get picked when.
<Tabs>
<TabItem value="rpm" label="RPM-based shuffling">
##### **LiteLLM Proxy Config.yaml**
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-v-2
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
rpm: 900
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-functioncalling
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
rpm: 10
```
##### **Python SDK**
```python
from litellm import Router
import asyncio
model_list = [{ # list of model deployments
"model_name": "gpt-3.5-turbo", # model alias
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-2", # actual model name
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"rpm": 900, # requests per minute for this API
}
}, {
"model_name": "gpt-3.5-turbo",
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-functioncalling",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"rpm": 10,
}
},]
# init router
router = Router(model_list=model_list, routing_strategy="simple-shuffle")
async def router_acompletion():
response = await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}]
)
print(response)
return response
asyncio.run(router_acompletion())
```
</TabItem>
<TabItem value="weight" label="Weight-based shuffling">
##### **LiteLLM Proxy Config.yaml**
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-v-2
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
weight: 9
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-functioncalling
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
weight: 1
```
##### **Python SDK**
```python
from litellm import Router
import asyncio
model_list = [{
"model_name": "gpt-3.5-turbo", # model alias
"litellm_params": {
"model": "azure/chatgpt-v-2", # actual model name
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"weight": 9, # pick this 90% of the time
}
}, {
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/chatgpt-functioncalling",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"weight": 1,
}
}]
# init router
router = Router(model_list=model_list, routing_strategy="simple-shuffle")
async def router_acompletion():
response = await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}]
)
print(response)
return response
asyncio.run(router_acompletion())
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="usage-based" label="Rate-Limit Aware">
This will route to the deployment with the lowest TPM usage for that minute.

2
docs/my-website/docs/scheduler.md
View file

@ -41,7 +41,7 @@ router = Router(
},
],
timeout=2, # timeout request if takes > 2s
routing_strategy="usage-based-routing-v2",
routing_strategy="simple-shuffle", # recommended for best performance
polling_interval=0.03 # poll queue every 3ms if no healthy deployments
)

1353
docs/my-website/docs/simple_proxy_old_doc.md
View file

File diff suppressed because it is too large Load diff

107
docs/my-website/docs/tutorials/claude_responses_api.md
View file

@ -4,14 +4,20 @@ import TabItem from '@theme/TabItem';
# Claude Code
This tutorial shows how to call the Responses API models like `codex-mini` and `o3-pro` from the Claude Code endpoint on LiteLLM.
This tutorial shows how to call Claude models through LiteLLM proxy from Claude Code.
:::info
This tutorial is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). This integration allows you to use any LiteLLM supported model through Claude Code.
This tutorial is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). This integration allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
:::
<br />
### Video Walkthrough
<iframe width="840" height="500" src="https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## Prerequisites
- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
@ -31,19 +37,18 @@ Create a secure configuration using environment variables:
```yaml
model_list:
# Responses API models
- model_name: codex-mini
# Claude models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: o3-pro
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
@ -51,7 +56,7 @@ litellm_settings:
Set your environment variables:
```bash
export OPENAI_API_KEY="your-openai-api-key"
export ANTHROPIC_API_KEY="your-anthropic-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
@ -72,31 +77,49 @@ curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "codex-mini",
"model": "claude-3-5-sonnet-20241022",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
### 4. Configure Claude Code
Setup Claude Code to use your LiteLLM proxy:
#### Method 1: Unified Endpoint (Recommended)
Configure Claude Code to use LiteLLM's unified endpoint:
Either a virtual key / master key can be used here
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
### 5. Use Claude Code
:::tip
LITELLM_MASTER_KEY gives claude access to all proxy models, whereas a virtual key would be limited to the models set in UI
:::
Start Claude Code with any configured model:
#### Method 2: Provider-specific Pass-through Endpoint
Alternatively, use the Anthropic pass-through endpoint:
```bash
# Use Responses API models
claude --model codex-mini
claude --model o3-pro
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
# Or use the latest model alias
claude --model codex-mini-latest
### 5. Use Claude Code
Start Claude Code and it will automatically use your configured models:
```bash
# Claude Code will use the models configured in your LiteLLM proxy
claude
# Or specify a model if you have multiple configured
claude --model claude-3-5-sonnet-20241022
claude --model claude-3-5-haiku-20241022
```
Example conversation:
@ -112,7 +135,8 @@ Common issues and solutions:
**Authentication errors:**
- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
- Check that your OpenAI API key is valid and has sufficient credits
- Check that your API keys are valid and have sufficient credits
- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Model not found:**
- Ensure the model name in Claude Code matches exactly with your `config.yaml`
@ -123,33 +147,47 @@ Common issues and solutions:
Expand your configuration to support multiple providers and models:
<Tabs>
<TabItem value="responses-plus" label="Responses + Standard Models">
<TabItem value="multi-provider" label="Multi-Provider Setup">
```yaml
model_list:
# Responses API models
# OpenAI models
- model_name: codex-mini
litellm_params:
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: o3-pro
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
# Standard models
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: claude-3-5-sonnet
# Anthropic models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
@ -158,13 +196,14 @@ litellm_settings:
Switch between models seamlessly:
```bash
# Use Responses API models for advanced reasoning
claude --model o3-pro
claude --model codex-mini
# Use Claude for complex reasoning
claude --model claude-3-5-sonnet-20241022
# Use standard models for general tasks
claude --model gpt-4o
claude --model claude-3-5-sonnet
# Use Haiku for fast responses
claude --model claude-3-5-haiku-20241022
# Use Bedrock deployment
claude --model claude-bedrock
```
</TabItem>

1
docs/my-website/docs/tutorials/scim_litellm.md
View file

@ -72,6 +72,7 @@ On the LiteLLM UI, Navigate to `Teams`, You should see the new team `Production
<Image img={require('../../img/msft_auto_team.png')} style={{ width: '900px', height: 'auto' }} />
> **Note:** When a user is removed from your organization via SCIM, all API keys and access tokens associated with that user will be automatically deleted from LiteLLM. This ensures that removed users lose all access immediately and securely.

BIN
docs/my-website/img/admin_settings_ui_theme.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 14 KiB

BIN
docs/my-website/img/admin_settings_ui_theme_logo.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 42 KiB

BIN
docs/my-website/img/create_team_member_rate_limits.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 100 KiB

BIN
docs/my-website/img/release_notes/team_member_rate_limits.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 372 KiB

6
docs/my-website/release_notes/v1.74.15-stable/index.md
View file

@ -86,9 +86,9 @@ This is great to central AI Platform teams looking to track how they are helping
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Cost per Image |
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------------- |
| OpenRouter | `openrouter/x-ai/grok-4` | 256k | $3 | $15 | N/A |
| Google AI Studio | `gemini/imagen-4.0-generate-preview-06-06` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-4.0-ultra-generate-preview-06-06` | N/A | N/A | N/A | $0.06 |
| Google AI Studio | `gemini/imagen-4.0-fast-generate-preview-06-06` | N/A | N/A | N/A | $0.02 |
| Google AI Studio | `gemini/imagen-4.0-generate-001` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-4.0-ultra-generate-001` | N/A | N/A | N/A | $0.06 |
| Google AI Studio | `gemini/imagen-4.0-fast-generate-001` | N/A | N/A | N/A | $0.02 |
| Google AI Studio | `gemini/imagen-3.0-generate-002` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-3.0-generate-001` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-3.0-fast-generate-001` | N/A | N/A | N/A | $0.02 |

1
docs/my-website/release_notes/v1.74.7/index.md
View file

@ -148,7 +148,6 @@ Starting with this release, you can run health endpoints on an isolated process
- New provider integration for v0.dev - [PR #12751](https://github.com/BerriAI/litellm/pull/12751), [Get Started](../../docs/providers/v0)
- **[OpenAI](../../docs/providers/openai)**
- Use OpenAI DeepResearch models with `litellm.completion` (`/chat/completions`) - [PR #12627](https://github.com/BerriAI/litellm/pull/12627) **DOC NEEDED**
- Add `input_fidelity` parameter for OpenAI image generation - [PR #12662](https://github.com/BerriAI/litellm/pull/12662), [Get Started](../../docs/image_generation)
- **[Azure OpenAI](../../docs/providers/azure_openai)**
- Use Azure OpenAI DeepResearch models with `litellm.completion` (`/chat/completions`) - [PR #12627](https://github.com/BerriAI/litellm/pull/12627) **DOC NEEDED**
- Added `response_format` support for openai gpt-4.1 models - [PR #12745](https://github.com/BerriAI/litellm/pull/12745)

20
docs/my-website/release_notes/v1.75.8/index.md
View file

@ -1,5 +1,5 @@
---
title: "[PRE-RELEASE]v1.75.8"
title: "v1.75.8-stable - Team Member Rate Limits"
slug: "v1-75-8"
date: 2025-08-16T10:00:00
authors:
@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.75.8
ghcr.io/berriai/litellm:v1.75.8-stable
```
</TabItem>
@ -52,6 +52,22 @@ pip install litellm==1.75.8
---
## Team Member Rate Limits
<Image
img={require('../../img/release_notes/team_member_rate_limits.png')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<p style={{textAlign: 'left', color: '#666'}}>
LiteLLM MCP Architecture: Use MCP tools with all LiteLLM supported models
</p>
This release adds support for setting rate limits on individual members (including machine users) within a team. Teams can now give each agent its own rate limits—so that heavy-traffic agents don’t impact other agents or human users.
Agents can authenticate with LiteLLM using JWT and the same team role as human users, while still enforcing per-agent rate limits.
## New Models / Updated Models
#### New Model Support

189
docs/my-website/release_notes/v1.76.0-stable/index.md Normal file
View file

@ -0,0 +1,189 @@
---
title: "[PRE-RELEASE]v1.76.0-stable - RPS Improvements"
slug: "v1-76-0"
date: 2025-08-23T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
:::info
LiteLLM is hiring a **Founding Backend Engineer**, in San Francisco.
[Apply here](https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer) if you're interested!
:::
## Deploy this version
:::info
This release is not live yet.
:::
---
## New Models / Updated Models
#### Bugs
- **[OpenAI](../../docs/providers/openai)**
- Gpt-5 chat: clarify does not support function calling [PR #13612](https://github.com/BerriAI/litellm/pull/13612), s/o  @[superpoussin22](https://github.com/superpoussin22)
- **[VertexAI](../../docs/providers/vertex)**
- fix vertexai batch file format by @[thiagosalvatore](https://github.com/thiagosalvatore) in [PR #13576](https://github.com/BerriAI/litellm/pull/13576)
- **[LiteLLM Proxy](../../docs/providers/litellm_proxy)**
- Add support for calling image_edits + image_generations via SDK to Proxy - [PR #13735](https://github.com/BerriAI/litellm/pull/13735)
- **[OpenRouter](../../docs/providers/openrouter)**
- Fix max_output_tokens value for anthropic Claude 4 - [PR #13526](https://github.com/BerriAI/litellm/pull/13526)
- **[Gemini](../../docs/providers/gemini)**
- Fix prompt caching cost calculation - [PR #13742](https://github.com/BerriAI/litellm/pull/13742)
- **[Azure](../../docs/providers/azure)**
- Support `../openai/v1/respones` api base - [PR #13526](https://github.com/BerriAI/litellm/pull/13526)
- Fix azure/gpt-5-chat max_input_tokens - [PR #13660](https://github.com/BerriAI/litellm/pull/13660)
- **[Groq](../../docs/providers/groq)**
- streaming ASCII encoding issue - [PR #13675](https://github.com/BerriAI/litellm/pull/13675)
- **[Baseten](../../docs/providers/baseten)**
- Refactored integration to use new openai-compatible endpoints - [PR #13783](https://github.com/BerriAI/litellm/pull/13783)
- **[Bedrock](../../docs/providers/bedrock)**
- fix application inference profile for pass-through endpoints for bedrock - [PR #13881](https://github.com/BerriAI/litellm/pull/13881)
- **[DataRobot](../../docs/providers/datarobot)**
- Updated URL handling for DataRobot provider URL - [PR #13880](https://github.com/BerriAI/litellm/pull/13880)
#### Features
- **[Together AI](../../docs/providers/together)**
- Added Qwen3, Deepseek R1 0528 Throughput, GLM 4.5 and GPT-OSS models cost tracking - [PR #13637](https://github.com/BerriAI/litellm/pull/13637), s/o  @[Tasmay-Tibrewal](https://github.com/Tasmay-Tibrewal)
- **[Fireworks AI](../../docs/providers/fireworks_ai)**
- add fireworks_ai/accounts/fireworks/models/deepseek-v3-0324 - [PR #13821](https://github.com/BerriAI/litellm/pull/13821)
- **[VertexAI](../../docs/providers/vertex)**
- Add VertexAI qwen API Service - [PR #13828](https://github.com/BerriAI/litellm/pull/13828)
- Add new VertexAI image models vertex_ai/imagen-4.0-generate-001, vertex_ai/imagen-4.0-ultra-generate-001, vertex_ai/imagen-4.0-fast-generate-001  - [PR #13874](https://github.com/BerriAI/litellm/pull/13874)
- **[Anthropic](../../docs/providers/anthropic)**
- Add long context support w/ cost tracking - [PR #13759](https://github.com/BerriAI/litellm/pull/13759)
- **[DeepInfra](../../docs/providers/deepinfra)**
- Add rerank endpoint support for deepinfra - [PR #13820](https://github.com/BerriAI/litellm/pull/13820)
- Add new models for cost tracking - [PR #13883](https://github.com/BerriAI/litellm/pull/13883), s/o  @[Toy-97](https://github.com/Toy-97)
- **[Bedrock](../../docs/providers/bedrock)**
- Add tool prompt caching on async calls - [PR #13803](https://github.com/BerriAI/litellm/pull/13803), s/o  @[UlookEE](https://github.com/UlookEE)
- role chaining and session name with webauthentication for aws bedrock - [PR #13753](https://github.com/BerriAI/litellm/pull/13753), s/o @[RichardoC](https://github.com/RichardoC)
- **[Ollama](../../docs/providers/ollama)**
- Handle Ollama null response when using tool calling with non-tool trained models - [PR #13902](https://github.com/BerriAI/litellm/pull/13902)
- **[OpenRouter](../../docs/providers/openrouter)**
- Add deepseek/deepseek-chat-v3.1 support - [PR #13897](https://github.com/BerriAI/litellm/pull/13897)
- **[Mistral](../../docs/providers/mistral)**
- Add support for calling mistral files via chat completions - [PR #13866](https://github.com/BerriAI/litellm/pull/13866), s/o  @[jinskjoy](https://github.com/jinskjoy)
- Handle empty assistant content - [PR #13671](https://github.com/BerriAI/litellm/pull/13671)
- Support new ‘thinking’ response block - [PR #13671](https://github.com/BerriAI/litellm/pull/13671)
- **[Databricks](../../docs/providers/databricks)**
- remove deprecated dbrx models (dbrx-instruct, llama 3.1) - [PR #13843](https://github.com/BerriAI/litellm/pull/13843)
- **[AI/ML API](../../docs/providers/ai_ml_api)**
- Image gen api support - [PR #13893](https://github.com/BerriAI/litellm/pull/13893)
## LLM API Endpoints
#### Bugs
- **[Responses API](../../docs/response_api)**
- add default api version for openai responses api calls - [PR #13526](https://github.com/BerriAI/litellm/pull/13526)
- support allowed_openai_params - [PR #13671](https://github.com/BerriAI/litellm/pull/13671)
## MCP Gateway
#### Bugs
- fix StreamableHTTPSessionManager .run() error - [PR #13666](https://github.com/BerriAI/litellm/pull/13666)
## Vector Stores
#### Bugs
- **[Bedrock](../../docs/providers/bedrock)**
- Using LiteLLM Managed Credentials for Query - [PR #13787](https://github.com/BerriAI/litellm/pull/13787)
## Management Endpoints / UI
#### Bugs
- **[Passthrough](../../docs/pass_through/intro)**
- Fix query passthrough deletion - [PR #13622](https://github.com/BerriAI/litellm/pull/13622)
#### Features
- **Models**
- Add Search Functionality for Public Model Names in Model Dashboard - [PR #13687](https://github.com/BerriAI/litellm/pull/13687)
- Auto-Add `azure/` to deployment Name in UI - [PR #13685](https://github.com/BerriAI/litellm/pull/13685)
- Models page row UI restructure - [PR #13771](https://github.com/BerriAI/litellm/pull/13771)
- **Notifications**
- Add new notifications toast UI everywhere - [PR #13813](https://github.com/BerriAI/litellm/pull/13813)
- **Keys**
- Fix key edit settings after regenerating a key - [PR #13815](https://github.com/BerriAI/litellm/pull/13815)
- Require team_id when creating service account keys - [PR #13873](https://github.com/BerriAI/litellm/pull/13873)
- Filter - show all options on filter option click - [PR #13858](https://github.com/BerriAI/litellm/pull/13858)
- **Usage**
- Fix ‘Cannot read properties of undefined’ exception on user agent activity tab - [PR #13892](https://github.com/BerriAI/litellm/pull/13892)
- **SSO**
- Free SSO usage for up to 5 users - [PR #13843](https://github.com/BerriAI/litellm/pull/13843)
## Logging / Guardrail Integrations
#### Bugs
- **[Bedrock Guardrails](../../docs/proxy/guardrails/bedrock)**
- Add bedrock api key support - [PR #13835](https://github.com/BerriAI/litellm/pull/13835)
#### Features
- **[Datadog LLM Observability](../../docs/integrations/datadog)**
- Add support for Failure Logging [PR #13726](https://github.com/BerriAI/litellm/pull/13726)
- Add time to first token, litellm overhead, guardrail overhead latency metrics - [PR #13734](https://github.com/BerriAI/litellm/pull/13734)
- Add support for tracing guardrail input/output - [PR #13767](https://github.com/BerriAI/litellm/pull/13767)
- **[Langfuse OTEL](../../docs/integrations/langfuse)**
- Allow using Key/Team Based Logging - [PR #13791](https://github.com/BerriAI/litellm/pull/13791)
- **[AIM](../../docs/integrations/aim)**
- Migrate to new firewall API - [PR #13748](https://github.com/BerriAI/litellm/pull/13748)
- **[OTEL](../../docs/observability/opentelemetry_integration)**
- Add OTEL tracing for actual LLM API call - [PR #13836](https://github.com/BerriAI/litellm/pull/13836)
- **[MLFlow](../../docs/observability/mlflow_integration)**
- Include predicted output in MLflow tracing - [PR #13795](https://github.com/BerriAI/litellm/pull/13795), s/o @TomeHirata 
## Performance / Loadbalancing / Reliability improvements
#### Bugs
- **[Cooldowns](../../docs/routing#how-cooldowns-work)**
- don't return raw Azure Exceptions to client (can contain prompt leakage) - [PR #13529](https://github.com/BerriAI/litellm/pull/13529)
- **[Auto-router](../../docs/proxy/auto_routing)**
- Ensures the relevant dependencies for auto router existing on LiteLLM Docker - [PR #13788](https://github.com/BerriAI/litellm/pull/13788)
- **Model Alias**
- Fix calling key with access to model alias - [PR #13830](https://github.com/BerriAI/litellm/pull/13830)
#### Features
- **[S3 Caching](../../docs/proxy/caching)**
- Use namespace as prefix for s3 cache - [PR #13704](https://github.com/BerriAI/litellm/pull/13704)
- Async S3 Caching support (4x RPS improvement) - [PR #13852](https://github.com/BerriAI/litellm/pull/13852), s/o @[michal-otmianowski](https://github.com/michal-otmianowski)
- **Model Group header forwarding**
- reuse same logic as global header forwarding - [PR #13741](https://github.com/BerriAI/litellm/pull/13741)
- add support for hosted_vllm on UI - [PR #13885](https://github.com/BerriAI/litellm/pull/13885)
- **Performance**
- Improve LiteLLM Python SDK RPS by +200 RPS (braintrust import + aiohttp transport fixes) - [PR #13839](https://github.com/BerriAI/litellm/pull/13839)
- Use O(1) Set lookups for model routing - [PR #13879](https://github.com/BerriAI/litellm/pull/13879)
- Reduce Significant CPU overhead from litellm_logging.py - [PR #13895](https://github.com/BerriAI/litellm/pull/13895)
- Improvements for Async Success Handler (Logging Callbacks) - Approx +130 RPS - [PR #13905](https://github.com/BerriAI/litellm/pull/13905)
## General Proxy Improvements
#### Bugs
- **SDK**
- Fix litellm compatibility with newest release of openAI (>v1.100.0) - [PR #13728](https://github.com/BerriAI/litellm/pull/13728)
- **Helm**
- Add possibility to configure resources for migrations-job - [PR #13617](https://github.com/BerriAI/litellm/pull/13617)
- Ensure Helm chart auto generated master keys follow sk-xxxx format - [PR #13871](https://github.com/BerriAI/litellm/pull/13871)
- Enhance database configuration: add support for optional endpointKey - [PR #13763](https://github.com/BerriAI/litellm/pull/13763)
- **Rate Limits**
- fixing descriptor/response size mismatch on parallel_request_limiter_v3 - [PR #13863](https://github.com/BerriAI/litellm/pull/13863), s/o  @[luizrennocosta](https://github.com/luizrennocosta)
- **Non-root**
- fix permission access on prisma migrate in non-root image - [PR #13848](https://github.com/BerriAI/litellm/pull/13848), s/o @[Ithanil](https://github.com/Ithanil)

269
docs/my-website/release_notes/v1.76.1-stable/index.md Normal file
View file

@ -0,0 +1,269 @@
---
title: "v1.76.1-stable - Gemini 2.5 Flash Image"
slug: "v1-76-1"
date: 2025-08-30T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.76.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.76.1
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Major Performance Improvements** - 6.5x faster LiteLLM Python SDK completion with fastuuid integration.
- **New Model Support** - Gemini 2.5 Flash Image Preview, Grok Code Fast, and GPT Realtime models
- **Enhanced Provider Support** - DeepSeek-v3.1 pricing on Fireworks AI, Vercel AI Gateway, and improved Anthropic/GitHub Copilot integration
- **MCP Improvements** - Better connection testing and SSE MCP tools bug fixes
## Major Changes
- Added support for using Gemini 2.5 Flash Image Preview with /chat/completions. **🚨 Warning** If you were using `gemini-2.0-flash-exp-image-generation` please follow this migration guide.
[Gemini Image Generation Migration Guide](../../docs/extras/gemini_img_migration)
---
## Performance Improvements
This release includes significant performance optimizations:
- **6.5x faster LiteLLM Python SDK Completion** - Major performance boost for completion operations - [PR #13990](https://github.com/BerriAI/litellm/pull/13990)
- **fastuuid Integration** - 2.1x faster UUID generation with +80 RPS improvement for /chat/completions and other LLM endpoints - [PR #13992](https://github.com/BerriAI/litellm/pull/13992), [PR #14016](https://github.com/BerriAI/litellm/pull/14016)
- **Optimized Request Logging** - Don't print request params by default for +50 RPS improvement - [PR #14015](https://github.com/BerriAI/litellm/pull/14015)
- **Cache Performance** - 21% speedup in InMemoryCache.evict_cache and 45% speedup in `_is_debugging_on` function - [PR #14012](https://github.com/BerriAI/litellm/pull/14012), [PR #13988](https://github.com/BerriAI/litellm/pull/13988)
---
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------- |
| Google | `gemini-2.5-flash-image-preview` | 1M | $0.30 | $2.50 | Chat completions + image generation ($0.039/image) |
| X.AI | `xai/grok-code-fast` | 256K | $0.20 | $1.50 | Code generation |
| OpenAI | `gpt-realtime` | 32K | $4.00 | $16.00 | Real-time conversation + audio |
| Vercel AI Gateway | `vercel_ai_gateway/openai/o3` | 200K | $2.00 | $8.00 | Advanced reasoning |
| Vercel AI Gateway | `vercel_ai_gateway/openai/o3-mini` | 200K | $1.10 | $4.40 | Efficient reasoning |
| Vercel AI Gateway | `vercel_ai_gateway/openai/o4-mini` | 200K | $1.10 | $4.40 | Latest mini model |
| DeepInfra | `deepinfra/zai-org/GLM-4.5` | 131K | $0.55 | $2.00 | Chat completions |
| Perplexity | `perplexity/codellama-34b-instruct` | 16K | $0.35 | $1.40 | Code generation |
| Fireworks AI | `fireworks_ai/accounts/fireworks/models/deepseek-v3p1` | 128K | $0.56 | $1.68 | Chat completions |
**Additional Models Added:** Various other Vercel AI Gateway models were added too. See [models.litellm.ai](https://models.litellm.ai) for the full list.
#### Features
- **[Google Gemini](../../docs/providers/gemini)**
- Added support for `gemini-2.5-flash-image-preview` with image return capability - [PR #13979](https://github.com/BerriAI/litellm/pull/13979), [PR #13983](https://github.com/BerriAI/litellm/pull/13983)
- Support for requests with only system prompt - [PR #14010](https://github.com/BerriAI/litellm/pull/14010)
- Fixed invalid model name error for Gemini Imagen models - [PR #13991](https://github.com/BerriAI/litellm/pull/13991)
- **[X.AI](../../docs/providers/xai)**
- Added `xai/grok-code-fast` model family support - [PR #14054](https://github.com/BerriAI/litellm/pull/14054)
- Fixed frequency_penalty parameter for grok-4 models - [PR #14078](https://github.com/BerriAI/litellm/pull/14078)
- **[OpenAI](../../docs/providers/openai)**
- Added support for gpt-realtime models - [PR #14082](https://github.com/BerriAI/litellm/pull/14082)
- Support for reasoning and reasoning_effort parameters by default - [PR #12865](https://github.com/BerriAI/litellm/pull/12865)
- **[Fireworks AI](../../docs/providers/fireworks_ai)**
- Added DeepSeek-v3.1 pricing - [PR #13958](https://github.com/BerriAI/litellm/pull/13958)
- **[DeepInfra](../../docs/providers/deepinfra)**
- Fixed reasoning_effort setting for DeepSeek-V3.1 - [PR #14053](https://github.com/BerriAI/litellm/pull/14053)
- **[GitHub Copilot](../../docs/providers/github_copilot)**
- Added support for thinking and reasoning_effort parameters - [PR #13691](https://github.com/BerriAI/litellm/pull/13691)
- Added image headers support - [PR #13955](https://github.com/BerriAI/litellm/pull/13955)
- **[Anthropic](../../docs/providers/anthropic)**
- Support for custom Anthropic-compatible API endpoints - [PR #13945](https://github.com/BerriAI/litellm/pull/13945)
- Fixed /messages fallback from Anthropic API to Bedrock API - [PR #13946](https://github.com/BerriAI/litellm/pull/13946)
- **[Nebius](../../docs/providers/nebius)**
- Expanded provider models and normalized model IDs - [PR #13965](https://github.com/BerriAI/litellm/pull/13965)
- **[Vertex AI](../../docs/providers/vertex)**
- Fixed Vertex Mistral streaming issues - [PR #13952](https://github.com/BerriAI/litellm/pull/13952)
- Fixed anyOf corner cases for Gemini tool calls - [PR #12797](https://github.com/BerriAI/litellm/pull/12797)
- **[Bedrock](../../docs/providers/bedrock)**
- Fixed structure output issues - [PR #14005](https://github.com/BerriAI/litellm/pull/14005)
- **[OpenRouter](../../docs/providers/openrouter)**
- Added GPT-5 family models pricing - [PR #13536](https://github.com/BerriAI/litellm/pull/13536)
#### New Provider Support
- **[Vercel AI Gateway](../../docs/providers/vercel_ai_gateway)**
- New provider support added - [PR #13144](https://github.com/BerriAI/litellm/pull/13144)
- **[DataRobot](../../docs/providers/datarobot)**
- Added provider documentation - [PR #14038](https://github.com/BerriAI/litellm/pull/14038), [PR #14074](https://github.com/BerriAI/litellm/pull/14074)
---
## LLM API Endpoints
#### Features
- **[Images API](../../docs/image_generation)**
- Support for multiple images in OpenAI images/edits endpoint - [PR #13916](https://github.com/BerriAI/litellm/pull/13916)
- Allow using dynamic `api_key` for image generation requests - [PR #14007](https://github.com/BerriAI/litellm/pull/14007)
- **[Responses API](../../docs/response_api)**
- Fixed `/responses` endpoint ignoring extra_headers in GitHub Copilot - [PR #13775](https://github.com/BerriAI/litellm/pull/13775)
- Added support for new web_search tool - [PR #14083](https://github.com/BerriAI/litellm/pull/14083)
- **[Azure Passthrough](../../docs/providers/azure/azure)**
- Fixed Azure Passthrough request with streaming - [PR #13831](https://github.com/BerriAI/litellm/pull/13831)
#### Bugs
- **General**
- Fixed handling of None metadata in batch requests - [PR #13996](https://github.com/BerriAI/litellm/pull/13996)
- Fixed token_counter with special token input - [PR #13374](https://github.com/BerriAI/litellm/pull/13374)
- Removed incorrect web search support for azure/gpt-4.1 family - [PR #13566](https://github.com/BerriAI/litellm/pull/13566)
---
## [MCP Gateway](../../docs/mcp)
#### Features
- **SSE MCP Tools**
- Bug fix for adding SSE MCP tools - improved connection testing when adding MCPs - [PR #14048](https://github.com/BerriAI/litellm/pull/14048)
[Read More](../../docs/mcp)
---
## Management Endpoints / UI
#### Features
- **Team Management**
- Allow setting Team Member RPM/TPM limits when creating a team - [PR #13943](https://github.com/BerriAI/litellm/pull/13943)
- **UI Improvements**
- Fixed Next.js Security Vulnerabilities in UI Dashboard - [PR #14084](https://github.com/BerriAI/litellm/pull/14084)
- Fixed collapsible navbar design - [PR #14075](https://github.com/BerriAI/litellm/pull/14075)
#### Bugs
- **Authentication**
- Fixed Virtual keys with llm_api type causing Internal Server Error for /anthropic/* and other LLM passthrough routes - [PR #14046](https://github.com/BerriAI/litellm/pull/14046)
---
## Logging / Guardrail Integrations
#### Features
- **[Langfuse OTEL](../../docs/proxy/logging#langfuse)**
- Allow using LANGFUSE_OTEL_HOST for configuring host - [PR #14013](https://github.com/BerriAI/litellm/pull/14013)
- **[Braintrust](../../docs/proxy/logging#braintrust)**
- Added span name metadata feature - [PR #13573](https://github.com/BerriAI/litellm/pull/13573)
- Fixed tests to reference moved attributes in `braintrust_logging` module - [PR #13978](https://github.com/BerriAI/litellm/pull/13978)
- **[OpenMeter](../../docs/proxy/logging#openmeter)**
- Set user from token user_id for OpenMeter integration - [PR #13152](https://github.com/BerriAI/litellm/pull/13152)
#### New Guardrail Support
- **[Noma Security](../../docs/proxy/guardrails)**
- Added Noma Security guardrail support - [PR #13572](https://github.com/BerriAI/litellm/pull/13572)
- **[Pangea](../../docs/proxy/guardrails)**
- Updated Pangea Guardrail to support new AIDR endpoint - [PR #13160](https://github.com/BerriAI/litellm/pull/13160)
---
## Performance / Loadbalancing / Reliability improvements
#### Features
- **Caching**
- Verify if cache entry has expired prior to serving it to client - [PR #13933](https://github.com/BerriAI/litellm/pull/13933)
- Fixed error saving latency as timedelta on Redis - [PR #14040](https://github.com/BerriAI/litellm/pull/14040)
- **Router**
- Refactored router to choose weights by 'weight', 'rpm', 'tpm' in one loop for simple_shuffle - [PR #13562](https://github.com/BerriAI/litellm/pull/13562)
- **Logging**
- Fixed LoggingWorker graceful shutdown to prevent CancelledError warnings - [PR #14050](https://github.com/BerriAI/litellm/pull/14050)
- Enhanced logging for containers to log on files both with usual format and json format - [PR #13394](https://github.com/BerriAI/litellm/pull/13394)
#### Bugs
- **Dependencies**
- Bumped `orjson` version to "3.11.2" - [PR #13969](https://github.com/BerriAI/litellm/pull/13969)
---
## General Proxy Improvements
#### Features
- **AWS**
- Add support for AWS assume_role with a session token - [PR #13919](https://github.com/BerriAI/litellm/pull/13919)
- **OCI Provider**
- Added oci_key_file as an optional_parameter - [PR #14036](https://github.com/BerriAI/litellm/pull/14036)
- **Configuration**
- Allow configuration to set threshold before request entry in spend log gets truncated - [PR #14042](https://github.com/BerriAI/litellm/pull/14042)
- Enhanced proxy_config configuration: add support for existing configmap in Helm charts - [PR #14041](https://github.com/BerriAI/litellm/pull/14041)
- **Docker**
- Added back supervisor to non-root image - [PR #13922](https://github.com/BerriAI/litellm/pull/13922)
---
## New Contributors
* @ArthurRenault made their first contribution in [PR #13922](https://github.com/BerriAI/litellm/pull/13922)
* @stevenmanton made their first contribution in [PR #13919](https://github.com/BerriAI/litellm/pull/13919)
* @uc4w6c made their first contribution in [PR #13914](https://github.com/BerriAI/litellm/pull/13914)
* @nielsbosma made their first contribution in [PR #13573](https://github.com/BerriAI/litellm/pull/13573)
* @Yuki-Imajuku made their first contribution in [PR #13567](https://github.com/BerriAI/litellm/pull/13567)
* @codeflash-ai[bot] made their first contribution in [PR #13988](https://github.com/BerriAI/litellm/pull/13988)
* @ColeFrench made their first contribution in [PR #13978](https://github.com/BerriAI/litellm/pull/13978)
* @dttran-glo made their first contribution in [PR #13969](https://github.com/BerriAI/litellm/pull/13969)
* @manascb1344 made their first contribution in [PR #13965](https://github.com/BerriAI/litellm/pull/13965)
* @DorZion made their first contribution in [PR #13572](https://github.com/BerriAI/litellm/pull/13572)
* @edwardsamuel made their first contribution in [PR #13536](https://github.com/BerriAI/litellm/pull/13536)
* @blahgeek made their first contribution in [PR #13374](https://github.com/BerriAI/litellm/pull/13374)
* @Deviad made their first contribution in [PR #13394](https://github.com/BerriAI/litellm/pull/13394)
* @XSAM made their first contribution in [PR #13775](https://github.com/BerriAI/litellm/pull/13775)
* @KRRT7 made their first contribution in [PR #14012](https://github.com/BerriAI/litellm/pull/14012)
* @ikaadil made their first contribution in [PR #13991](https://github.com/BerriAI/litellm/pull/13991)
* @timelfrink made their first contribution in [PR #13691](https://github.com/BerriAI/litellm/pull/13691)
* @qidu made their first contribution in [PR #13562](https://github.com/BerriAI/litellm/pull/13562)
* @nagyv made their first contribution in [PR #13243](https://github.com/BerriAI/litellm/pull/13243)
* @xywei made their first contribution in [PR #12885](https://github.com/BerriAI/litellm/pull/12885)
* @ericgtkb made their first contribution in [PR #12797](https://github.com/BerriAI/litellm/pull/12797)
* @NoWall57 made their first contribution in [PR #13945](https://github.com/BerriAI/litellm/pull/13945)
* @lmwang9527 made their first contribution in [PR #14050](https://github.com/BerriAI/litellm/pull/14050)
* @WilsonSunBritten made their first contribution in [PR #14042](https://github.com/BerriAI/litellm/pull/14042)
* @Const-antine made their first contribution in [PR #14041](https://github.com/BerriAI/litellm/pull/14041)
* @dmvieira made their first contribution in [PR #14040](https://github.com/BerriAI/litellm/pull/14040)
* @gotsysdba made their first contribution in [PR #14036](https://github.com/BerriAI/litellm/pull/14036)
* @moshemorad made their first contribution in [PR #14005](https://github.com/BerriAI/litellm/pull/14005)
* @joshualipman123 made their first contribution in [PR #13144](https://github.com/BerriAI/litellm/pull/13144)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.76.0-nightly...v1.76.1)**

289
docs/my-website/release_notes/v1.76.3-stable/index.md Normal file
View file

@ -0,0 +1,289 @@
---
title: "v1.76.3-stable - Performance, Video Generation & CloudZero Integration"
slug: "v1-76-3"
date: 2025-09-06T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
:::warning
This release has a known issue where startup is leading to Out of Memory errors when deploying on Kubernetes. We recommend waiting before upgrading to this version.
:::
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.76.3
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.76.3
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Major Performance Improvements** +400 RPS when using correct amount of workers + CPU cores combination
- **Video Generation Support** - Added Google AI Studio and Vertex AI Veo Video Generation through LiteLLM Pass through routes
- **CloudZero Integration** - New cost tracking integration for exporting LiteLLM Usage and Spend data to CloudZero.
## Major Changes
- **Performance Optimization**: LiteLLM Proxy now achieves +400 RPS when using correct amount of CPU cores - [PR #14153](https://github.com/BerriAI/litellm/pull/14153), [PR #14242](https://github.com/BerriAI/litellm/pull/14242)
By default, LiteLLM will now use `num_workers = os.cpu_count()` to achieve optimal performance.
**Override Options:**
Set environment variable:
```bash
DEFAULT_NUM_WORKERS_LITELLM_PROXY=1
```
Or start LiteLLM Proxy with:
```bash
litellm --num_workers 1
```
- **Security Fix**: Fixed memory_usage_in_mem_cache cache endpoint vulnerability - [PR #14229](https://github.com/BerriAI/litellm/pull/14229)
---
## Performance Improvements
This release includes significant performance optimizations. On our internal benchmarks we saw 1 instance get +400 RPS when using correct amount of workers + CPU cores combination.
- **+400 RPS Performance Boost** - LiteLLM Proxy now uses correct amount of CPU cores for optimal performance - [PR #14153](https://github.com/BerriAI/litellm/pull/14153)
- **Default CPU Workers** - Changed DEFAULT_NUM_WORKERS_LITELLM_PROXY default to number of CPUs - [PR #14242](https://github.com/BerriAI/litellm/pull/14242)
---
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------- |
| OpenRouter | `openrouter/openai/gpt-4.1` | 1M | $2.00 | $8.00 | Chat completions with vision |
| OpenRouter | `openrouter/openai/gpt-4.1-mini` | 1M | $0.40 | $1.60 | Efficient chat completions |
| OpenRouter | `openrouter/openai/gpt-4.1-nano` | 1M | $0.10 | $0.40 | Ultra-efficient chat |
| Vertex AI | `vertex_ai/openai/gpt-oss-20b-maas` | 131K | $0.075 | $0.30 | Reasoning support |
| Vertex AI | `vertex_ai/openai/gpt-oss-120b-maas` | 131K | $0.15 | $0.60 | Advanced reasoning |
| Gemini | `gemini/veo-3.0-generate-preview` | 1K | - | $0.75/sec | Video generation |
| Gemini | `gemini/veo-3.0-fast-generate-preview` | 1K | - | $0.40/sec | Fast video generation |
| Gemini | `gemini/veo-2.0-generate-001` | 1K | - | $0.35/sec | Video generation |
| Volcengine | `doubao-embedding-large` | 4K | Free | Free | 2048-dim embeddings |
| Together AI | `together_ai/deepseek-ai/DeepSeek-V3.1` | 128K | $0.60 | $1.70 | Reasoning support |
#### Features
- **[Google Gemini](../../docs/providers/gemini)**
- Added 'thoughtSignature' support via 'thinking_blocks' - [PR #14122](https://github.com/BerriAI/litellm/pull/14122)
- Added support for reasoning_effort='minimal' for Gemini models - [PR #14262](https://github.com/BerriAI/litellm/pull/14262)
- **[OpenRouter](../../docs/providers/openrouter)**
- Added GPT-4.1 model family - [PR #14101](https://github.com/BerriAI/litellm/pull/14101)
- **[Groq](../../docs/providers/groq)**
- Added support for reasoning_effort parameter - [PR #14207](https://github.com/BerriAI/litellm/pull/14207)
- **[X.AI](../../docs/providers/xai)**
- Fixed XAI cost calculation - [PR #14127](https://github.com/BerriAI/litellm/pull/14127)
- **[Vertex AI](../../docs/providers/vertex)**
- Added support for GPT-OSS models on Vertex AI - [PR #14184](https://github.com/BerriAI/litellm/pull/14184)
- Added additionalProperties to Vertex AI Schema definition - [PR #14252](https://github.com/BerriAI/litellm/pull/14252)
- **[VLLM](../../docs/providers/vllm)**
- Handle output parsing responses API output - [PR #14121](https://github.com/BerriAI/litellm/pull/14121)
- **[Ollama](../../docs/providers/ollama)**
- Added unified 'thinking' param support via `reasoning_content` - [PR #14121](https://github.com/BerriAI/litellm/pull/14121)
- **[Anthropic](../../docs/providers/anthropic)**
- Added supported text field to anthropic citation response - [PR #14126](https://github.com/BerriAI/litellm/pull/14126)
- **[OCI Provider](../../docs/providers/oci)**
- Handle assistant messages with both content and tool_calls - [PR #14171](https://github.com/BerriAI/litellm/pull/14171)
- **[Bedrock](../../docs/providers/bedrock)**
- Fixed structure output - [PR #14130](https://github.com/BerriAI/litellm/pull/14130)
- Added initial support for Bedrock Batches API - [PR #14190](https://github.com/BerriAI/litellm/pull/14190)
- **[Databricks](../../docs/providers/databricks)**
- Added support for anthropic citation API in Databricks - [PR #14077](https://github.com/BerriAI/litellm/pull/14077)
### Bug Fixes
- **[Google Gemini (Google AI Studio + Vertex AI)](../../docs/providers/gemini)**
- Fixed Gemini 2.5 Pro schema validation with OpenAI-style type arrays in tools - [PR #14154](https://github.com/BerriAI/litellm/pull/14154)
- Fixed Gemini Tool Calling empty enum property - [PR #14155](https://github.com/BerriAI/litellm/pull/14155)
#### New Provider Support
- **[Volcengine](../../docs/providers/volcengine)**
- Added Volcengine embedding module with handler and transformation logic - [PR #14028](https://github.com/BerriAI/litellm/pull/14028)
---
## LLM API Endpoints
#### Features
- **[Images API](../../docs/image_generation)**
- Added pass through image generation and image editing on OpenAI - [PR #14292](https://github.com/BerriAI/litellm/pull/14292)
- Support extra_body parameter for image generation - [PR #14211](https://github.com/BerriAI/litellm/pull/14211)
- **[Responses API](../../docs/response_api)**
- Fixed response API for reasoning item in input for litellm proxy - [PR #14200](https://github.com/BerriAI/litellm/pull/14200)
- Added structured output for SDK - [PR #14206](https://github.com/BerriAI/litellm/pull/14206)
- **[Bedrock Passthrough](../../docs/pass_through/bedrock)**
- Support AWS_BEDROCK_RUNTIME_ENDPOINT on bedrock passthrough - [PR #14156](https://github.com/BerriAI/litellm/pull/14156)
- **[Google AI Studio Passthrough](../../docs/pass_through/google_ai_studio)**
- Allow using Veo Video Generation through LiteLLM Pass through routes - [PR #14228](https://github.com/BerriAI/litellm/pull/14228)
- **General**
- Added support for safety_identifier parameter in chat.completions.create - [PR #14174](https://github.com/BerriAI/litellm/pull/14174)
- Fixed misclassified 500 error on invalid image_url in /chat/completions request - [PR #14149](https://github.com/BerriAI/litellm/pull/14149)
- Fixed token count error for Gemini CLI - [PR #14133](https://github.com/BerriAI/litellm/pull/14133)
#### Bugs
- **General**
- Remove "/" or ":" from model name when being used as h11 header name - [PR #14191](https://github.com/BerriAI/litellm/pull/14191)
- Bug fix for openai.gpt-oss when using reasoning_effort parameter - [PR #14300](https://github.com/BerriAI/litellm/pull/14300)
---
## Spend Tracking, Budgets and Rate Limiting
### Features
- Added header support for spend_logs_metadata - [PR #14186](https://github.com/BerriAI/litellm/pull/14186)
- Litellm passthrough cost tracking for chat completion - [PR #14256](https://github.com/BerriAI/litellm/pull/14256)
### Bug Fixes
- Fixed TPM Rate Limit Bug - [PR #14237](https://github.com/BerriAI/litellm/pull/14237)
- Fixed Key Budget not resets at expectable times - [PR #14241](https://github.com/BerriAI/litellm/pull/14241)
## Management Endpoints / UI
#### Features
- **UI Improvements**
- Logs page screen size fixed - [PR #14135](https://github.com/BerriAI/litellm/pull/14135)
- Create Organization Tooltip added on Success - [PR #14132](https://github.com/BerriAI/litellm/pull/14132)
- Back to Keys should say Back to Logs - [PR #14134](https://github.com/BerriAI/litellm/pull/14134)
- Add client side pagination on All Models table - [PR #14136](https://github.com/BerriAI/litellm/pull/14136)
- Model Filters UI improvement - [PR #14131](https://github.com/BerriAI/litellm/pull/14131)
- Remove table filter on user info page - [PR #14169](https://github.com/BerriAI/litellm/pull/14169)
- Team name badge added on the User Details - [PR #14003](https://github.com/BerriAI/litellm/pull/14003)
- Fix: Log page parameter passing error - [PR #14193](https://github.com/BerriAI/litellm/pull/14193)
- **Authentication & Authorization**
- Support for ES256/ES384/ES512 and EdDSA JWT verification - [PR #14118](https://github.com/BerriAI/litellm/pull/14118)
- Ensure `team_id` is a required field for generating service account keys - [PR #14270](https://github.com/BerriAI/litellm/pull/14270)
#### Bugs
- **General**
- Validate store model in db setting - [PR #14269](https://github.com/BerriAI/litellm/pull/14269)
---
## Logging / Guardrail Integrations
#### Features
- **[Datadog](../../docs/proxy/logging#datadog)**
- Ensure `apm_id` is set on DD LLM Observability traces - [PR #14272](https://github.com/BerriAI/litellm/pull/14272)
- **[Braintrust](../../docs/proxy/logging#braintrust)**
- Fix logging when OTEL is enabled - [PR #14122](https://github.com/BerriAI/litellm/pull/14122)
- **[OTEL](../../docs/proxy/logging#otel)**
- Optional Metrics and Logs following semantic conventions - [PR #14179](https://github.com/BerriAI/litellm/pull/14179)
- **[Slack Alerting](../../docs/proxy/alerting)**
- Added alert type to alert message to slack for easier handling - [PR #14176](https://github.com/BerriAI/litellm/pull/14176)
#### Guardrails
- Added guardrail to the Anthropic API endpoint - [PR #14107](https://github.com/BerriAI/litellm/pull/14107)
#### New Integration
- **[CloudZero](../../docs/proxy/cost_tracking)**
- LiteLLM x CloudZero Integration for Cost Tracking - [PR #14296](https://github.com/BerriAI/litellm/pull/14296)
---
## Performance / Loadbalancing / Reliability improvements
#### Features
- **Performance**
- LiteLLM Proxy: +400 RPS when using correct amount of CPU cores - [PR #14153](https://github.com/BerriAI/litellm/pull/14153)
- Allow using `x-litellm-stream-timeout` header for stream timeout in requests - [PR #14147](https://github.com/BerriAI/litellm/pull/14147)
- Change DEFAULT_NUM_WORKERS_LITELLM_PROXY default to number CPUs - [PR #14242](https://github.com/BerriAI/litellm/pull/14242)
- **Monitoring**
- Added Prometheus missing metrics - [PR #14139](https://github.com/BerriAI/litellm/pull/14139)
- **Timeout**
- **Stream Timeout Control** - Allow using `x-litellm-stream-timeout` header for stream timeout in requests - [PR #14147](https://github.com/BerriAI/litellm/pull/14147)
- **Routing**
- Fixed x-litellm-tags not routing with Responses API - [PR #14289](https://github.com/BerriAI/litellm/pull/14289)
#### Bugs
- **Security**
- Fixed memory_usage_in_mem_cache cache endpoint vulnerability - [PR #14229](https://github.com/BerriAI/litellm/pull/14229)
---
## General Proxy Improvements
#### Features
- **SCIM Support**
- Added better SCIM debugging - [PR #14221](https://github.com/BerriAI/litellm/pull/14221)
- Bug fixes for handling SCIM Group Memberships - [PR #14226](https://github.com/BerriAI/litellm/pull/14226)
- **Kubernetes**
- Added optional PodDisruptionBudget for litellm proxy - [PR #14093](https://github.com/BerriAI/litellm/pull/14093)
- **Error Handling**
- Add model to azure error message - [PR #14294](https://github.com/BerriAI/litellm/pull/14294)
---
## New Contributors
* @iabhi4 made their first contribution in [PR #14093](https://github.com/BerriAI/litellm/pull/14093)
* @zainhas made their first contribution in [PR #14087](https://github.com/BerriAI/litellm/pull/14087)
* @LifeDJIK made their first contribution in [PR #14146](https://github.com/BerriAI/litellm/pull/14146)
* @retanoj made their first contribution in [PR #14133](https://github.com/BerriAI/litellm/pull/14133)
* @zhxlp made their first contribution in [PR #14193](https://github.com/BerriAI/litellm/pull/14193)
* @kayoch1n made their first contribution in [PR #14191](https://github.com/BerriAI/litellm/pull/14191)
* @kutsushitaneko made their first contribution in [PR #14171](https://github.com/BerriAI/litellm/pull/14171)
* @mjmendo made their first contribution in [PR #14176](https://github.com/BerriAI/litellm/pull/14176)
* @HarshavardhanK made their first contribution in [PR #14213](https://github.com/BerriAI/litellm/pull/14213)
* @eycjur made their first contribution in [PR #14207](https://github.com/BerriAI/litellm/pull/14207)
* @22mSqRi made their first contribution in [PR #14241](https://github.com/BerriAI/litellm/pull/14241)
* @onlylhf made their first contribution in [PR #14028](https://github.com/BerriAI/litellm/pull/14028)
* @btpemercier made their first contribution in [PR #11319](https://github.com/BerriAI/litellm/pull/11319)
* @tremlin made their first contribution in [PR #14287](https://github.com/BerriAI/litellm/pull/14287)
* @TobiMayr made their first contribution in [PR #14262](https://github.com/BerriAI/litellm/pull/14262)
* @Eitan1112 made their first contribution in [PR #14252](https://github.com/BerriAI/litellm/pull/14252)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.76.1-nightly...v1.76.3-nightly)**

Some files were not shown because too many files have changed in this diff Show more