scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-21 00:21:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 74

Merge branch 'main' into litellm_performance_infra_setup_000001

Browse source
This commit is contained in:
Alexsander Hamir 2026-01-30 09:50:30 -08:00 • committed by GitHub
commit f4af124ebb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
643 changed files with 33791 additions and 4226 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

153
.circleci/config.yml
View file

@ -715,8 +715,8 @@ jobs:
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_router_coverage.xml
mv .coverage litellm_router_coverage
mv coverage.xml litellm_router_unit_coverage.xml
mv .coverage litellm_router_unit_coverage
# Store test results
- store_test_results:
path: test-results
@ -724,8 +724,8 @@ jobs:
- persist_to_workspace:
root: .
paths:
- litellm_router_coverage.xml
- litellm_router_coverage
- litellm_router_unit_coverage.xml
- litellm_router_unit_coverage
litellm_security_tests:
machine:
image: ubuntu-2204:2023.10.1
@ -3407,6 +3407,110 @@ jobs:
- store_test_results:
path: test-results
proxy_e2e_anthropic_messages_tests:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.10
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.10 -y
conda activate myenv
python --version
- run:
name: Install Dependencies
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
pip install "pytest==7.3.1"
pip install "pytest-asyncio==0.21.1"
pip install "boto3==1.36.0"
pip install "httpx==0.27.0"
pip install "claude-agent-sdk"
pip install -r requirements.txt
- run:
name: Install dockerize
command: |
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container with test config
command: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME="us-east-1" \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
- run:
name: Start outputting logs
command: docker logs -f my-app
background: true
- run:
name: Wait for app to be ready
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Run Claude Agent SDK E2E Tests
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
pwd
ls
python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
# Store test results
- store_test_results:
path: test-results
upload-coverage:
docker:
- image: cimg/python:3.9
@ -3428,7 +3532,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@ -3478,8 +3582,22 @@ jobs:
ls dist/
twine upload --verbose dist/*
else
echo "Version ${VERSION} of package is already published on PyPI. Skipping PyPI publish."
circleci step halt
echo "Version ${VERSION} of package is already published on PyPI."
# Check if corresponding Docker nightly image exists
NIGHTLY_TAG="v${VERSION}-nightly"
echo "Checking for Docker nightly image: litellm/litellm:${NIGHTLY_TAG}"
# Check Docker Hub for the nightly image
if curl -s "https://hub.docker.com/v2/repositories/litellm/litellm/tags/${NIGHTLY_TAG}" | grep -q "name"; then
echo "Docker nightly image ${NIGHTLY_TAG} exists. This release was already completed successfully."
echo "Skipping PyPI publish and continuing to ensure Docker images are up to date."
circleci step halt
else
echo "ERROR: PyPI package ${VERSION} exists but Docker nightly image ${NIGHTLY_TAG} does not exist!"
echo "This indicates an incomplete release. Please investigate."
exit 1
fi
fi
- run:
name: Trigger Github Action for new Docker Container + Trigger Load Testing
@ -3488,11 +3606,21 @@ jobs:
python3 -m pip install toml
VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
echo "LiteLLM Version ${VERSION}"
# Determine which branch to use for Docker build
if [[ "$CIRCLE_BRANCH" =~ ^litellm_release_day_.* ]]; then
BUILD_BRANCH="$CIRCLE_BRANCH"
echo "Using release branch: $BUILD_BRANCH"
else
BUILD_BRANCH="main"
echo "Using default branch: $BUILD_BRANCH"
fi
curl -X POST \
-H "Accept: application/vnd.github.v3+json" \
-H "Authorization: Bearer $GITHUB_TOKEN" \
"https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \
-d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
-d "{\"ref\":\"${BUILD_BRANCH}\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
echo "triggering load testing server for version ${VERSION} and commit ${CIRCLE_SHA1}"
curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}&release_type=nightly"
@ -4051,6 +4179,14 @@ workflows:
only:
- main
- /litellm_.*/
- proxy_e2e_anthropic_messages_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- llm_translation_testing:
filters:
branches:
@ -4235,6 +4371,7 @@ workflows:
branches:
only:
- main
- /litellm_release_day_.*/
- publish_to_pypi:
requires:
- mypy_linting

3
.circleci/requirements.txt
View file

@ -16,4 +16,5 @@ uvloop==0.21.0
mcp==1.25.0 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
fastuuid==0.12.0
responses==0.25.7 # for proxy client tests
responses==0.25.7 # for proxy client tests
pytest-retry==1.6.3 # for automatic test retries

2
.github/workflows/test-linting.yml vendored
View file

@ -73,4 +73,4 @@ jobs:
- name: Check import safety
run: |
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)

2
.github/workflows/test-litellm.yml vendored
View file

@ -34,7 +34,7 @@ jobs:
poetry run pip install "google-genai==1.22.0"
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.18"
poetry run pip install "python-multipart==0.0.22"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |

15
.github/workflows/test-model-map.yaml vendored Normal file
View file

@ -0,0 +1,15 @@
name: Validate model_prices_and_context_window.json
on:
pull_request:
branches: [ main ]
jobs:
validate-model-prices-json:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Validate model_prices_and_context_window.json
run: |
jq empty model_prices_and_context_window.json

8
.gitignore vendored
View file

@ -60,10 +60,6 @@ litellm/proxy/_super_secret_config.yaml
litellm/proxy/myenv/bin/activate
litellm/proxy/myenv/bin/Activate.ps1
myenv/*
litellm/proxy/_experimental/out/_next/
litellm/proxy/_experimental/out/404/index.html
litellm/proxy/_experimental/out/model_hub/index.html
litellm/proxy/_experimental/out/onboarding/index.html
litellm/tests/log.txt
litellm/tests/langfuse.log
litellm/tests/langfuse.log
@ -76,9 +72,6 @@ tests/local_testing/log.txt
litellm/proxy/_new_new_secret_config.yaml
litellm/proxy/custom_guardrail.py
.mypy_cache/*
litellm/proxy/_experimental/out/404.html
litellm/proxy/_experimental/out/404.html
litellm/proxy/_experimental/out/model_hub.html
.mypy_cache/*
litellm/proxy/application.log
tests/llm_translation/vertex_test_account.json
@ -100,7 +93,6 @@ litellm_config.yaml
litellm/proxy/to_delete_loadtest_work/*
update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
litellm/proxy/_experimental/out/guardrails/index.html
scripts/test_vertex_ai_search.py
LAZY_LOADING_IMPROVEMENTS.md
**/test-results

8
AGENTS.md
View file

@ -51,12 +51,14 @@ LiteLLM is a unified interface for 100+ LLMs that:
### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
1. **Use Common Components as much as possible**:
1. **Tremor is DEPRECATED, do not use Tremor components in new features/changes**
- The only exception is the Tremor Table component and its required Tremor Table sub components.
2. **Use Common Components as much as possible**:
- These are usually defined in the `common_components` directory
- Use these components as much as possible and avoid building new components unless needed
- Tremor components are deprecated; prefer using Ant Design (AntD) as much as possible
2. **Testing**:
3. **Testing**:
- The codebase uses **Vitest** and **React Testing Library**
- **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
- **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)

8
Dockerfile
View file

@ -46,8 +46,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
# Ensure runtime stage runs as root
USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -69,8 +69,8 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Generate prisma client
RUN prisma generate
# Generate prisma client using the correct schema
RUN prisma generate --schema=./litellm/proxy/schema.prisma
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh

1
README.md
View file

@ -267,6 +267,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
<td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
<td><h2>Netflix</h2></td>
<td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
</tr>
</table>

117
cookbook/anthropic_agent_sdk/README.md Normal file
View file

@ -0,0 +1,117 @@
# Claude Agent SDK with LiteLLM Gateway
A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK.
## Quick Start
### 1. Install dependencies
```bash
pip install anthropic claude-agent-sdk litellm
```
### 2. Start LiteLLM proxy
```bash
# Simple start with Claude
litellm --model claude-sonnet-4-20250514
# Or with a config file
litellm --config config.yaml
```
### 3. Run the chat
```bash
python main.py
```
That's it! You can now chat with the agent in your terminal.
### Chat Commands
While chatting, you can use these commands:
- `models` - List all available models (fetched from your LiteLLM proxy)
- `model` - Switch to a different model
- `clear` - Start a new conversation
- `quit` or `exit` - End the chat
The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured.
## Configuration
Set these environment variables if needed:
```bash
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
export LITELLM_MODEL="claude-sonnet-4-20250514"
```
Or just use the defaults - it'll connect to `http://localhost:4000` by default.
## Example Config File
If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`):
```yaml
model_list:
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
```
Then start LiteLLM with: `litellm --config config.yaml`
## How It Works
The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
```python
# Point to LiteLLM gateway (not Anthropic)
os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key
# Use any model configured in LiteLLM
options = ClaudeAgentOptions(
model="bedrock-claude-sonnet-4", # or gpt-4, or anything else
system_prompt="You are a helpful assistant.",
max_turns=50,
)
```
Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically.
## Why Use This?
- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc.
- **Cost tracking**: LiteLLM tracks spending across all your agent conversations
- **Rate limiting**: Set budgets and limits on your agent usage
- **Load balancing**: Distribute requests across multiple API keys or regions
- **Fallbacks**: Automatically retry with a different model if one fails
## Troubleshooting
**Connection errors?**
- Make sure LiteLLM is running: `litellm --model your-model`
- Check the URL is correct (default: `http://localhost:4000`)
**Authentication errors?**
- Verify your LiteLLM API key is correct
- Make sure the model is configured in your LiteLLM setup
**Model not found?**
- Check the model name matches what's in your LiteLLM config
- Run `litellm --model your-model` to test it works
## Learn More
- [LiteLLM Docs](https://docs.litellm.ai/)
- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk)
- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start)

25
cookbook/anthropic_agent_sdk/config.example.yaml Normal file
View file

@ -0,0 +1,25 @@
model_list:
- model_name: bedrock-claude-sonnet-3.5
litellm_params:
model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-opus-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-nova-premier
litellm_params:
model: "bedrock/amazon.nova-premier-v1:0"
aws_region_name: "us-east-1"

196
cookbook/anthropic_agent_sdk/main.py Normal file
View file

@ -0,0 +1,196 @@
"""
Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway
This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy.
LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.)
through the Claude Agent SDK by pointing it to the LiteLLM gateway.
"""
import os
import asyncio
import httpx
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
class Config:
"""Configuration for LiteLLM Gateway connection"""
# LiteLLM proxy URL (default to local instance)
LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
# LiteLLM API key (master key or virtual key)
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
# Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
"""
Fetch available models from LiteLLM proxy /models endpoint
"""
try:
async with httpx.AsyncClient() as client:
response = await client.get(
f"{base_url}/models",
headers={"Authorization": f"Bearer {api_key}"},
timeout=10.0
)
response.raise_for_status()
data = response.json()
return [model["id"] for model in data.get("data", [])]
except Exception as e:
print(f"⚠️ Warning: Could not fetch models from proxy: {e}")
print("Using default model list...")
# Fallback to default models
return [
"bedrock-claude-sonnet-3.5",
"bedrock-claude-sonnet-4",
"bedrock-claude-sonnet-4.5",
"bedrock-claude-opus-4.5",
"bedrock-nova-premier",
]
async def interactive_chat():
"""
Interactive CLI chat with the agent
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
# Note: We don't add /anthropic to the base URL - LiteLLM handles routing
litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
current_model = config.LITELLM_MODEL
print("=" * 70)
print("🤖 Claude Agent SDK with LiteLLM Gateway - Interactive Chat")
print("=" * 70)
print(f"🚀 Connected to: {litellm_base_url}")
print(f"📦 Current model: {current_model}")
print("\nType your messages below. Commands:")
print(" - 'quit' or 'exit' to end the conversation")
print(" - 'clear' to start a new conversation")
print(" - 'model' to switch models")
print(" - 'models' to list available models")
print("=" * 70)
print()
while True:
# Configure agent options for each conversation
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
# Create agent client
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
user_input = input("\n👤 You: ").strip()
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
print("\n📋 Available models:")
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
continue
if user_input.lower() == 'model':
print("\n📋 Select a model:")
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
try:
choice = input("\nEnter number (or press Enter to cancel): ").strip()
if choice:
idx = int(choice) - 1
if 0 <= idx < len(available_models):
current_model = available_models[idx]
print(f"\n✅ Switched to: {current_model}")
print("🔄 Starting new conversation with new model...\n")
conversation_active = False
else:
print("❌ Invalid choice")
except (ValueError, IndexError):
print("❌ Invalid input")
continue
if not user_input:
continue
# Send query to agent with loading indicator
print("\n🤖 Assistant: ", end='', flush=True)
try:
await client.query(user_input)
# Show loading indicator
print("⏳ thinking...", end='', flush=True)
# Stream the response
first_chunk = True
async for msg in client.receive_response():
# Clear loading indicator on first message
if first_chunk:
print("\r🤖 Assistant: ", end='', flush=True)
first_chunk = False
# Handle different message types
if hasattr(msg, 'type'):
if msg.type == 'content_block_delta':
# Streaming text delta
if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
print(msg.delta.text, end='', flush=True)
elif msg.type == 'content_block_start':
# Start of content block
if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
print(msg.content_block.text, end='', flush=True)
# Fallback to original content handling
if hasattr(msg, 'content'):
for content_block in msg.content:
if hasattr(content_block, 'text'):
print(content_block.text, end='', flush=True)
print() # New line after response
except Exception as e:
print(f"\r\n❌ Error: {e}")
print("Please check your LiteLLM gateway is running and configured correctly.")
def main():
"""Run interactive chat"""
try:
asyncio.run(interactive_chat())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
if __name__ == "__main__":
main()

2
cookbook/anthropic_agent_sdk/requirements.txt Normal file
View file

@ -0,0 +1,2 @@
claude-agent-sdk
httpx>=0.27.0

4
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -38,6 +38,10 @@ spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- with .Values.extraInitContainers }}
initContainers:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: {{ include "litellm.name" . }}
securityContext:

4
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -35,6 +35,10 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.migrationJob.extraInitContainers }}
initContainers:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: prisma-migrations
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}"

1
deploy/charts/litellm-helm/values.yaml
View file

@ -281,6 +281,7 @@ migrationJob:
# cpu: 100m
# memory: 100Mi
extraContainers: []
extraInitContainers: []
# Hook configuration
hooks:

6
docker/Dockerfile.non_root
View file

@ -170,12 +170,14 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+rX $PRISMA_PATH && \
chmod -R g+rX /app/.cache && \
mkdir -p /tmp/.npm /nonexistent /.npm && \
prisma generate
mkdir -p /tmp/.npm /nonexistent /.npm
# Switch to non-root user for runtime
USER nobody
# Generate Prisma client as nobody user to ensure correct file ownership
RUN prisma generate
# Prisma runtime knobs for offline containers
ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
PRISMA_HIDE_UPDATE_MESSAGE=1 \

116
docs/my-website/docs/a2a.md
View file

@ -68,7 +68,7 @@ Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./pr
## Invoking your Agents
Use the [A2A Python SDK](https://pypi.org/project/a2a/) to invoke agents through LiteLLM.
Use the [A2A Python SDK](https://pypi.org/project/a2a-sdk) to invoke agents through LiteLLM.
This example shows how to:
1. **List available agents** - Query `/v1/agents` to see which agents your key can access
@ -193,6 +193,120 @@ The logs show:
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
## Forwarding LiteLLM Context Headers
When LiteLLM invokes your A2A agent, it sends special headers that enable:
- **Trace Grouping**: All LLM calls from the same agent execution appear under one trace
- **Agent Spend Tracking**: Costs are attributed to the specific agent
| Header | Purpose |
|--------|---------|
| `X-LiteLLM-Trace-Id` | Links all LLM calls to the same execution flow |
| `X-LiteLLM-Agent-Id` | Attributes spend to the correct agent |
To enable these features, your A2A server must **forward these headers** to any LLM calls it makes back to LiteLLM.
### Implementation Steps
**Step 1: Extract headers from incoming A2A request**
```python def get_litellm_headers(request) -> dict:
"""Extract X-LiteLLM-* headers from incoming A2A request."""
all_headers = request.call_context.state.get('headers', {})
return {
k: v for k, v in all_headers.items()
if k.lower().startswith('x-litellm-')
}
```
**Step 2: Forward headers to your LLM calls**
Pass the extracted headers when making calls back to LiteLLM:
<Tabs>
<TabItem value="openai" label="OpenAI SDK" default>
```python from openai import OpenAI
headers = get_litellm_headers(request)
client = OpenAI(
api_key="sk-your-litellm-key",
base_url="http://localhost:4000",
default_headers=headers, # Forward headers
)
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}]
)
```
</TabItem>
<TabItem value="langchain" label="LangChain">
```python
from langchain_openai import ChatOpenAI
headers = get_litellm_headers(request)
llm = ChatOpenAI(
model="gpt-4o",
openai_api_key="sk-your-litellm-key",
base_url="http://localhost:4000",
default_headers=headers, # Forward headers
)
```
</TabItem>
<TabItem value="litellm" label="LiteLLM SDK">
```python
import litellm
headers = get_litellm_headers(request)
response = litellm.completion(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}],
api_base="http://localhost:4000",
extra_headers=headers, # Forward headers
)
```
</TabItem>
<TabItem value="requests" label="HTTP (requests/httpx)">
```python
import httpx
headers = get_litellm_headers(request)
headers["Authorization"] = "Bearer sk-your-litellm-key"
response = httpx.post(
"http://localhost:4000/v1/chat/completions",
headers=headers,
json={"model": "gpt-4o", "messages": [{"role": "user", "content": "Hello"}]}
)
```
</TabItem>
</Tabs>
### Result
With header forwarding enabled, you'll see:
**Trace Grouping in Langfuse:**
<Image
img={require('../img/a2a_trace_grouping.png')}
style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
/>
**Agent Spend Attribution:**
<Image
img={require('../img/a2a_agent_spend.png')}
style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
/>
## API Reference
### Endpoint

52
docs/my-website/docs/observability/datadog.md
View file

@ -7,6 +7,7 @@ import TabItem from '@theme/TabItem';
LiteLLM Supports logging to the following Datdog Integrations:
- `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
- `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
- `datadog_cost_management` [Datadog Cloud Cost Management](#datadog-cloud-cost-management)
- `ddtrace-run` [Datadog Tracing](#datadog-tracing)
## Datadog Logs
@ -73,7 +74,7 @@ Send logs through a local DataDog agent (useful for containerized environments):
```shell
LITELLM_DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent
LITELLM_DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518)
DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth)
DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (Agent handles auth for Logs. REQUIRED for LLM Observability)
DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source
```
@ -84,6 +85,9 @@ When `LITELLM_DD_AGENT_HOST` is set, logs are sent to the agent instead of direc
**Note:** We use `LITELLM_DD_AGENT_HOST` instead of `DD_AGENT_HOST` to avoid conflicts with `ddtrace` which automatically sets `DD_AGENT_HOST` for APM tracing.
> [!IMPORTANT]
> **Datadog LLM Observability**: `DD_API_KEY` is **REQUIRED** even when using the Datadog Agent (`LITELLM_DD_AGENT_HOST`). The agent acts as a proxy but the API key header is mandatory for the LLM Observability endpoint.
**Step 3**: Start the proxy, make a test request
Start proxy
@ -161,6 +165,50 @@ On the Datadog LLM Observability page, you should see that both input messages a
<Image img={require('../../img/dd_llm_obs.png')} />
## Datadog Cloud Cost Management
| Feature | Details |
|---------|---------|
| **What is logged** | Aggregated LLM Costs (FOCUS format) |
| **Events** | Periodic Uploads of Aggregated Cost Data |
| **Product Link** | [Datadog Cloud Cost Management](https://docs.datadoghq.com/cost_management/) |
We will use the `--config` to set `litellm.callbacks = ["datadog_cost_management"]`. This will periodically upload aggregated LLM cost data to Datadog.
**Step 1**: Create a `config.yaml` file and set `litellm_settings`: `success_callback`
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
litellm_settings:
callbacks: ["datadog_cost_management"]
```
**Step 2**: Set Required env variables
```shell
DD_API_KEY="your-api-key"
DD_APP_KEY="your-app-key" # REQUIRED for Cost Management
DD_SITE="us5.datadoghq.com"
```
**Step 3**: Start the proxy
```shell
litellm --config config.yaml
```
**How it works**
* LiteLLM aggregates costs in-memory by Provider, Model, Date, and Tags.
* Requires `DD_APP_KEY` for the Custom Costs API.
* Costs are uploaded periodically (flushed).
### Datadog Tracing
Use `ddtrace-run` to enable [Datadog Tracing](https://ddtrace.readthedocs.io/en/stable/installation_quickstart.html) on litellm proxy
@ -203,5 +251,5 @@ LiteLLM supports customizing the following Datadog environment variables
| `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No |
\* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required
\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required
\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required for **Datadog Logs**. (**Note: `DD_API_KEY` IS REQUIRED for Datadog LLM Observability**)

25
docs/my-website/docs/pass_through/openai_passthrough.md
View file

@ -1,6 +1,6 @@
# OpenAI Passthrough
Pass-through endpoints for `/openai`
Pass-through endpoints for direct OpenAI API access
## Overview
@ -10,12 +10,27 @@ Pass-through endpoints for `/openai`
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | Fully supported |
### When to use this?
## Available Endpoints
### `/openai_passthrough` - Recommended
Dedicated passthrough endpoint that guarantees direct routing to OpenAI without conflicts.
**Use this for:**
- OpenAI Responses API (`/v1/responses`)
- Any endpoint where you need guaranteed passthrough
- When `/openai` routes are conflicting with LiteLLM's native implementations
### `/openai` - Legacy
Standard passthrough endpoint that may conflict with LiteLLM's native implementations.
**Note:** Some endpoints like `/openai/v1/responses` will be routed to LiteLLM's native implementation instead of OpenAI.
## When to use this?
- For 90% of your use cases, you should use the [native LiteLLM OpenAI Integration](https://docs.litellm.ai/docs/providers/openai) (`/chat/completions`, `/embeddings`, `/completions`, `/images`, `/batches`, etc.)
- Use this passthrough to call less popular or newer OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`
- Use `/openai_passthrough` to call less popular or newer OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`, `/responses`
Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai`
Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai_passthrough`
## Usage Examples
@ -34,7 +49,7 @@ Make sure you do the following:
import openai
client = openai.OpenAI(
base_url="http://0.0.0.0:4000/openai", # <your-proxy-url>/openai
base_url="http://0.0.0.0:4000/openai_passthrough", # <your-proxy-url>/openai_passthrough
api_key="sk-anything" # <your-proxy-api-key>
)
```

450
docs/my-website/docs/providers/anthropic_tool_search.md
View file

@ -1,43 +1,46 @@
# Anthropic Tool Search
# Tool Search
Tool search enables Claude to dynamically discover and load tools on-demand from large tool catalogs (10,000+ tools). Instead of loading all tool definitions into the context window upfront, Claude searches your tool catalog and loads only the tools it needs.
## Supported Providers
| Provider | Chat Completions API | Messages API |
|----------|---------------------|--------------|
| **Anthropic API** | ✅ | ✅ |
| **Azure Anthropic** (Microsoft Foundry) | ✅ | ✅ |
| **Google Cloud Vertex AI** | ✅ | ✅ |
| **Amazon Bedrock** | ✅ (Invoke API only, Opus 4.5 only) | ✅ (Invoke API only, Opus 4.5 only) |
## Benefits
- **Context efficiency**: Avoid consuming massive portions of your context window with tool definitions
- **Better tool selection**: Claude's tool selection accuracy degrades with more than 30-50 tools. Tool search maintains accuracy even with thousands of tools
- **On-demand loading**: Tools are only loaded when Claude needs them
## Supported Models
Tool search is available on:
- Claude Opus 4.5
- Claude Sonnet 4.5
## Supported Platforms
- Anthropic API (direct)
- Azure Anthropic (Microsoft Foundry)
- Google Cloud Vertex AI
- Amazon Bedrock (invoke API only, not converse API)
## Tool Search Variants
LiteLLM supports both tool search variants:
### 1. Regex Tool Search (`tool_search_tool_regex_20251119`)
Claude constructs regex patterns to search for tools.
Claude constructs regex patterns to search for tools. Best for exact pattern matching (faster).
### 2. BM25 Tool Search (`tool_search_tool_bm25_20251119`)
Claude uses natural language queries to search for tools using the BM25 algorithm.
Claude uses natural language queries to search for tools using the BM25 algorithm. Best for natural language semantic search.
## Quick Start
**Note**: BM25 variant is not supported on Bedrock.
### Basic Example with Regex Tool Search
---
```python
## Chat Completions API
### SDK Usage
#### Basic Example with Regex Tool Search
```python showLineNumbers title="Basic Tool Search Example"
import litellm
response = litellm.completion(
@ -70,26 +73,6 @@ response = litellm.completion(
}
},
"defer_loading": True # Mark for deferred loading
},
# Another deferred tool
{
"type": "function",
"function": {
"name": "search_files",
"description": "Search through files in the workspace",
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string"},
"file_types": {
"type": "array",
"items": {"type": "string"}
}
},
"required": ["query"]
}
},
"defer_loading": True
}
]
)
@ -97,9 +80,9 @@ response = litellm.completion(
print(response.choices[0].message.content)
```
### BM25 Tool Search Example
#### BM25 Tool Search Example
```python
```python showLineNumbers title="BM25 Tool Search"
import litellm
response = litellm.completion(
@ -134,9 +117,9 @@ response = litellm.completion(
)
```
## Using with Azure Anthropic
#### Azure Anthropic Example
```python
```python showLineNumbers title="Azure Anthropic Tool Search"
import litellm
response = litellm.completion(
@ -170,9 +153,9 @@ response = litellm.completion(
)
```
## Using with Vertex AI
#### Vertex AI Example
```python
```python showLineNumbers title="Vertex AI Tool Search"
import litellm
response = litellm.completion(
@ -192,11 +175,9 @@ response = litellm.completion(
)
```
## Streaming Support
#### Streaming Support
Tool search works with streaming:
```python
```python showLineNumbers title="Streaming with Tool Search"
import litellm
response = litellm.completion(
@ -233,13 +214,13 @@ for chunk in response:
print(chunk.choices[0].delta.content, end="")
```
## LiteLLM Proxy
### AI Gateway Usage
Tool search works automatically through the LiteLLM proxy:
Tool search works automatically through the LiteLLM proxy.
### Proxy Config
#### Proxy Configuration
```yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: claude-sonnet
litellm_params:
@ -247,18 +228,19 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
```
### Client Request
#### Client Request
```python
import openai
```python showLineNumbers title="Client Request via Proxy"
from anthropic import Anthropic
client = openai.OpenAI(
client = Anthropic(
api_key="your-litellm-proxy-key",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
response = client.messages.create(
model="claude-sonnet",
max_tokens=1024,
messages=[
{"role": "user", "content": "What's the weather?"}
],
@ -268,17 +250,14 @@ response = client.chat.completions.create(
"name": "tool_search_tool_regex"
},
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather information",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
"name": "get_weather",
"description": "Get weather information",
"input_schema": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
},
"defer_loading": True
}
@ -286,127 +265,278 @@ response = client.chat.completions.create(
)
```
## Important Notes
---
### Beta Header
## Messages API
LiteLLM automatically detects tool search tools and adds the appropriate beta header based on your provider:
The Messages API provides native Anthropic-style tool search support via the `litellm.anthropic.messages` interface.
- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20`
- **Google Cloud Vertex AI**: `tool-search-tool-2025-10-19`
- **Amazon Bedrock** (Invoke API, Opus 4.5 only): `tool-search-tool-2025-10-19`
### SDK Usage
You don't need to manually specify beta headers—LiteLLM handles this automatically.
#### Basic Example
### Deferred Loading
```python showLineNumbers title="Messages API - Basic Tool Search"
import litellm
- Tools with `defer_loading: true` are only loaded when Claude discovers them via search
- At least one tool must be non-deferred (the tool search tool itself)
- Keep your 3-5 most frequently used tools as non-deferred for optimal performance
### Tool Descriptions
Write clear, descriptive tool names and descriptions that match how users describe tasks. The search algorithm uses:
- Tool names
- Tool descriptions
- Argument names
- Argument descriptions
### Usage Tracking
Tool search requests are tracked in the usage object:
```python
response = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[{"role": "user", "content": "Search for tools"}],
tools=[...]
response = await litellm.anthropic.messages.acreate(
model="anthropic/claude-sonnet-4-20250514",
messages=[
{
"role": "user",
"content": "What's the weather in San Francisco?"
}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"name": "get_weather",
"description": "Get the current weather for a location",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA"
}
},
"required": ["location"]
},
"defer_loading": True
}
],
max_tokens=1024,
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
)
# Check tool search usage
if response.usage.server_tool_use:
print(f"Tool search requests: {response.usage.server_tool_use.tool_search_requests}")
print(response)
```
## Error Handling
#### Azure Anthropic Messages Example
### All Tools Deferred
```python showLineNumbers title="Azure Anthropic Messages API"
import litellm
```python
# ❌ This will fail - at least one tool must be non-deferred
tools = [
{
"type": "function",
"function": {...},
"defer_loading": True
}
]
# ✅ Correct - tool search tool is non-deferred
tools = [
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"type": "function",
"function": {...},
"defer_loading": True
}
]
response = await litellm.anthropic.messages.acreate(
model="azure_anthropic/claude-sonnet-4-20250514",
messages=[
{
"role": "user",
"content": "What's the stock price of Apple?"
}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"name": "get_stock_price",
"description": "Get the current stock price for a ticker symbol",
"input_schema": {
"type": "object",
"properties": {
"ticker": {
"type": "string",
"description": "The stock ticker symbol, e.g. AAPL"
}
},
"required": ["ticker"]
},
"defer_loading": True
}
],
max_tokens=1024,
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
)
```
### Missing Tool Definition
#### Vertex AI Messages Example
If Claude references a tool that isn't in your deferred tools list, you'll get an error. Make sure all tools that might be discovered are included in the tools parameter with `defer_loading: true`.
```python showLineNumbers title="Vertex AI Messages API"
import litellm
## Best Practices
response = await litellm.anthropic.messages.acreate(
model="vertex_ai/claude-sonnet-4@20250514",
messages=[
{
"role": "user",
"content": "Search the web for information about AI"
}
],
tools=[
{
"type": "tool_search_tool_bm25_20251119",
"name": "tool_search_tool_bm25"
},
{
"name": "search_web",
"description": "Search the web for information",
"input_schema": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "The search query"
}
},
"required": ["query"]
},
"defer_loading": True
}
],
max_tokens=1024,
extra_headers={"anthropic-beta": "tool-search-tool-2025-10-19"}
)
```
1. **Keep frequently used tools non-deferred**: Your 3-5 most common tools should not have `defer_loading: true`
#### Bedrock Messages Example
2. **Use semantic descriptions**: Tool descriptions should use natural language that matches user queries
```python showLineNumbers title="Bedrock Messages API (Invoke)"
import litellm
3. **Choose the right variant**:
- Use **regex** for exact pattern matching (faster)
- Use **BM25** for natural language semantic search
response = await litellm.anthropic.messages.acreate(
model="bedrock/invoke/anthropic.claude-opus-4-20250514-v1:0",
messages=[
{
"role": "user",
"content": "What's the weather?"
}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"name": "get_weather",
"description": "Get weather information",
"input_schema": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
},
"defer_loading": True
}
],
max_tokens=1024,
extra_headers={"anthropic-beta": "tool-search-tool-2025-10-19"}
)
```
4. **Monitor usage**: Track `tool_search_requests` in the usage object to understand search patterns
#### Streaming Support
5. **Optimize tool catalog**: Remove unused tools and consolidate similar functionality
```python showLineNumbers title="Messages API - Streaming"
import litellm
import json
## When to Use Tool Search
response = await litellm.anthropic.messages.acreate(
model="anthropic/claude-sonnet-4-20250514",
messages=[
{
"role": "user",
"content": "What's the weather in Tokyo?"
}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"name": "get_weather",
"description": "Get weather information",
"input_schema": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
},
"defer_loading": True
}
],
max_tokens=1024,
stream=True,
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
)
**Good use cases:**
- 10+ tools available in your system
- Tool definitions consuming >10K tokens
- Experiencing tool selection accuracy issues
- Building systems with multiple tool categories
- Tool library growing over time
async for chunk in response:
if isinstance(chunk, bytes):
chunk_str = chunk.decode("utf-8")
for line in chunk_str.split("\n"):
if line.startswith("data: "):
try:
json_data = json.loads(line[6:])
print(json_data)
except json.JSONDecodeError:
pass
```
**When traditional tool calling is better:**
- Less than 10 tools total
- All tools are frequently used
- Very small tool definitions (\<100 tokens total)
### AI Gateway Usage
## Limitations
Configure the proxy to use Messages API endpoints.
- Not compatible with tool use examples
- Requires Claude Opus 4.5 or Sonnet 4.5
- On Bedrock, only available via invoke API (not converse API)
- On Bedrock, only supported for Claude Opus 4.5 (not Sonnet 4.5)
- BM25 variant (`tool_search_tool_bm25_20251119`) is not supported on Bedrock
- Maximum 10,000 tools in catalog
- Returns 3-5 most relevant tools per search
#### Proxy Configuration
### Bedrock-Specific Notes
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: claude-sonnet-messages
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: os.environ/ANTHROPIC_API_KEY
```
When using Bedrock's Invoke API:
- The regex variant (`tool_search_tool_regex_20251119`) is automatically normalized to `tool_search_tool_regex`
- The BM25 variant (`tool_search_tool_bm25_20251119`) is automatically filtered out as it's not supported
- Tool search is only available for Claude Opus 4.5 models
#### Client Request
```python showLineNumbers title="Client Request via Proxy (Messages API)"
from anthropic import Anthropic
client = Anthropic(
api_key="your-litellm-proxy-key",
base_url="http://0.0.0.0:4000"
)
response = client.messages.create(
model="claude-sonnet-messages",
max_tokens=1024,
messages=[
{
"role": "user",
"content": "What's the weather?"
}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"name": "get_weather",
"description": "Get weather information",
"input_schema": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
},
"defer_loading": True
}
],
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
)
print(response)
```
---
## Additional Resources
- [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search)
- [LiteLLM Tool Calling Guide](https://docs.litellm.ai/docs/completion/function_call)

51
docs/my-website/docs/providers/gemini.md
View file

@ -1840,6 +1840,57 @@ content = response.get('choices', [{}])[0].get('message', {}).get('content')
print(content)
```
## gemini-robotics-er-1.5-preview Usage
```python
from litellm import api_base
from openai import OpenAI
import os
import base64
client = OpenAI(base_url="http://0.0.0.0:4000", api_key="sk-12345")
base64_image = base64.b64encode(open("closeup-object-on-table-many-260nw-1216144471.webp", "rb").read()).decode()
import json
import re
tools = [{"codeExecution": {}}]
response = client.chat.completions.create(
model="gemini/gemini-robotics-er-1.5-preview",
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "Point to no more than 10 items in the image. The label returned should be an identifying name for the object detected. The answer should follow the json format: [{\"point\": [y, x], \"label\": <label1>}, ...]. The points are in [y, x] format normalized to 0-1000."
},
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}
}
]
}
],
tools=tools
)
# Extract JSON from markdown code block if present
content = response.choices[0].message.content
# Look for triple-backtick JSON block
match = re.search(r'```json\s*(.*?)\s*```', content, re.DOTALL)
if match:
json_str = match.group(1)
else:
json_str = content
try:
data = json.loads(json_str)
print(json.dumps(data, indent=2))
except Exception as e:
print("Error parsing response as JSON:", e)
print("Response content:", content)
```
## Usage - PDF / Videos / etc. Files
### Inline Data (e.g. audio stream)

89
docs/my-website/docs/providers/sarvam.md Normal file
View file

@ -0,0 +1,89 @@
# Sarvam.ai
LiteLLM supports all the text models from [Sarvam ai](https://docs.sarvam.ai/api-reference-docs/chat/chat-completions)
## Usage
```python
import os
from litellm import completion
# Set your Sarvam API key
os.environ["SARVAM_API_KEY"] = ""
messages = [{"role": "user", "content": "Hello"}]
response = completion(
model="sarvam/sarvam-m",
messages=messages,
)
print(response)
```
## Usage with LiteLLM Proxy Server
Here's how to call a Sarvam.ai model with the LiteLLM Proxy Server
1. **Modify the `config.yaml`:**
```yaml
model_list:
- model_name: my-model
litellm_params:
model: sarvam/<your-model-name> # add sarvam/ prefix to route as Sarvam provider
api_key: api-key # api key to send your model
```
2. **Start the proxy:**
```bash
$ litellm --config /path/to/config.yaml
```
3. **Send a request to LiteLLM Proxy Server:**
<Tabs>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
base_url="http://0.0.0.0:4000" # litellm-proxy-base url
)
response = client.chat.completions.create(
model="my-model",
messages=[
{
"role": "user",
"content": "what llm are you"
}
],
)
print(response)
```
</TabItem>
<TabItem value="curl" label="curl">
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "my-model",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>

36
docs/my-website/docs/providers/vercel_ai_gateway.md
View file

@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem';
| Provider Route on LiteLLM | `vercel_ai_gateway/` |
| Link to Provider Doc | [Vercel AI Gateway Documentation ↗](https://vercel.com/docs/ai-gateway) |
| Base URL | `https://ai-gateway.vercel.sh/v1` |
| Supported Operations | `/chat/completions`, `/models` |
| Supported Operations | `/chat/completions`, `/embeddings`, `/models` |
<br />
<br />
@ -73,7 +73,7 @@ messages = [{"content": "Hello, how are you?", "role": "user"}]
# Vercel AI Gateway call with streaming
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
model="vercel_ai_gateway/openai/gpt-4o",
messages=messages,
stream=True
)
@ -82,6 +82,33 @@ for chunk in response:
print(chunk)
```
### Embeddings
```python showLineNumbers title="Vercel AI Gateway Embeddings"
import os
from litellm import embedding
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
# Vercel AI Gateway embedding call
response = embedding(
model="vercel_ai_gateway/openai/text-embedding-3-small",
input="Hello world"
)
print(response.data[0]["embedding"][:5]) # Print first 5 dimensions
```
You can also specify the `dimensions` parameter:
```python showLineNumbers title="Vercel AI Gateway Embeddings with Dimensions"
response = embedding(
model="vercel_ai_gateway/openai/text-embedding-3-small",
input=["Hello world", "Goodbye world"],
dimensions=768
)
```
## Usage - LiteLLM Proxy
Add the following to your LiteLLM Proxy configuration file:
@ -97,6 +124,11 @@ model_list:
litellm_params:
model: vercel_ai_gateway/anthropic/claude-4-sonnet
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
- model_name: text-embedding-3-small-gateway
litellm_params:
model: vercel_ai_gateway/openai/text-embedding-3-small
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
```
Start your LiteLLM Proxy server:

31
docs/my-website/docs/proxy/cli_sso.md
View file

@ -28,6 +28,37 @@ EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
:::
### Configuration
#### JWT Token Expiration
By default, CLI authentication tokens expire after **24 hours**. You can customize this expiration time by setting the `LITELLM_CLI_JWT_EXPIRATION_HOURS` environment variable when starting your LiteLLM Proxy:
```bash
# Set CLI JWT tokens to expire after 48 hours
export LITELLM_CLI_JWT_EXPIRATION_HOURS=48
export EXPERIMENTAL_UI_LOGIN="True"
litellm --config config.yaml
```
Or in a single command:
```bash
LITELLM_CLI_JWT_EXPIRATION_HOURS=48 EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
```
**Examples:**
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=12` - Tokens expire after 12 hours
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=168` - Tokens expire after 7 days (168 hours)
- `LITELLM_CLI_JWT_EXPIRATION_HOURS=720` - Tokens expire after 30 days (720 hours)
:::tip
You can check your current token's age and expiration status using:
```bash
litellm-proxy whoami
```
:::
### Steps
1. **Install the CLI**

2
docs/my-website/docs/proxy/config_settings.md
View file

@ -464,6 +464,7 @@ router_settings:
| CHATGPT_USER_AGENT_SUFFIX | Suffix to append to the ChatGPT user agent string
| CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI
| CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
| CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours. Can also be set via LITELLM_CLI_JWT_EXPIRATION_HOURS
| CLOUDZERO_API_KEY | CloudZero API key for authentication
| CLOUDZERO_CONNECTION_ID | CloudZero connection ID for data submission
| CLOUDZERO_EXPORT_INTERVAL_MINUTES | Interval in minutes for CloudZero data export operations
@ -731,6 +732,7 @@ router_settings:
| LITERAL_API_URL | API URL for Literal service
| LITERAL_BATCH_SIZE | Batch size for Literal operations
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
| LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI

3
docs/my-website/docs/proxy/guardrails/onyx_security.md
View file

@ -128,6 +128,7 @@ guardrails:
mode: ["pre_call", "post_call", "during_call"] # Run at multiple stages
api_key: os.environ/ONYX_API_KEY
api_base: os.environ/ONYX_API_BASE
timeout: 10.0 # Optional, defaults to 10 seconds
```
### Required Parameters
@ -137,6 +138,7 @@ guardrails:
### Optional Parameters
- **`api_base`**: Onyx API base URL (defaults to `https://ai-guard.onyx.security`)
- **`timeout`**: Request timeout in seconds (defaults to `10.0`)
## Environment Variables
@ -145,4 +147,5 @@ You can set these environment variables instead of hardcoding values in your con
```shell
export ONYX_API_KEY="your-api-key-here"
export ONYX_API_BASE="https://ai-guard.onyx.security" # Optional
export ONYX_TIMEOUT=10 # Optional, timeout in seconds
```

6
docs/my-website/docs/proxy/guardrails/quick_start.md
View file

@ -405,14 +405,10 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
## **Proxy Admin Controls**
### ✨ Monitoring Guardrails
### Monitoring Guardrails
Monitor which guardrails were executed and whether they passed or failed. e.g. guardrail going rogue and failing requests we don't intend to fail
:::info
✨ This is an Enterprise only feature [Get a free trial](https://www.litellm.ai/enterprise#trial)
:::
#### Setup

150
docs/my-website/docs/proxy/keys_teams_router_settings.md Normal file
View file

@ -0,0 +1,150 @@
import Image from '@theme/IdealImage';
# UI - Router Settings for Keys and Teams
Configure router settings at the key and team level to achieve granular control over routing behavior, fallbacks, retries, and other router configurations. This enables you to customize routing behavior for specific keys or teams without affecting global settings.
## Overview
Router Settings for Keys and Teams allows you to configure router behavior at different levels of granularity. Previously, router settings could only be configured globally, applying the same routing strategy, fallbacks, timeouts, and retry policies to all requests across your entire proxy instance.
With key-level and team-level router settings, you can now:
- **Customize routing strategies** per key or team (e.g., use `least-busy` for high-priority keys, `latency-based-routing` for others)
- **Configure different fallback chains** for different keys or teams
- **Set key-specific or team-specific timeouts** and retry policies
- **Apply different reliability settings** (cooldowns, allowed failures) per key or team
- **Override global settings** when needed for specific use cases
<Image img={require('../../img/ui_granular_router_settings.png')} />
## Summary
Router settings follow a **hierarchical resolution order**: **Keys > Teams > Global**. When a request is made:
1. **Key-level settings** are checked first. If router settings are configured for the API key being used, those settings are applied.
2. **Team-level settings** are checked next. If the key belongs to a team and that team has router settings configured, those settings are used (unless key-level settings exist).
3. **Global settings** are used as the final fallback. If neither key nor team settings are found, the global router settings from your proxy configuration are applied.
This hierarchical approach ensures that the most specific settings take precedence, allowing you to fine-tune routing behavior for individual keys or teams while maintaining sensible defaults at the global level.
## How Router Settings Resolution Works
Router settings are resolved in the following priority order:
### Resolution Order: Key > Team > Global
1. **Key-level router settings** (highest priority)
- Applied when router settings are configured directly on an API key
- Takes precedence over all other settings
- Useful for individual key customization
2. **Team-level router settings** (medium priority)
- Applied when the API key belongs to a team with router settings configured
- Only used if no key-level settings exist
- Useful for applying consistent settings across multiple keys in a team
3. **Global router settings** (lowest priority)
- Applied from your proxy configuration file or database
- Used as the default when no key or team settings are found
- Previously, this was the only option available
## How to Configure Router Settings
### Configuring Router Settings for Keys
Follow these steps to configure router settings for an API key:
1. Navigate to [http://localhost:4000/ui/?login=success](http://localhost:4000/ui/?login=success)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/61889da3-32de-4ebf-9cf3-7dc1db2fc993/ascreenshot_2492cf6d916a4ab98197cc8336e3a371_text_export.jpeg)
2. Click "+ Create New Key" (or edit an existing key)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/61889da3-32de-4ebf-9cf3-7dc1db2fc993/ascreenshot_5a25380cf5044b4f93c146139d84403a_text_export.jpeg)
3. Click "Optional Settings"
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/e5eb5858-1cc1-4273-90bd-19ad139feebd/ascreenshot_33888989cfb9445bb83660f702ba32e0_text_export.jpeg)
4. Click "Router Settings"
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/d9eeca83-1f76-4fcf-bf61-d89edf3454d3/ascreenshot_825c7993f4b24949aee9b31d4a788d8a_text_export.jpeg)
5. Configure your desired router settings. For example, click "Fallbacks" to configure fallback models:
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/30ff647f-0254-4410-8311-660eef7ec0c4/ascreenshot_16966c8a0160473eb03e0f2c3b5c3afa_text_export.jpeg)
6. Click "Select a model to begin configuring fallbacks" and configure your fallback chain:
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/918f1b5b-c656-4864-98bd-d8c58924b6d9/ascreenshot_79ca6cd93be04033929f080e0c8d040a_text_export.jpeg)
### Configuring Router Settings for Teams
Follow these steps to configure router settings for a team:
1. Navigate to [http://localhost:4000/ui/?login=success](http://localhost:4000/ui/?login=success)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/60a33a8c-2e48-4788-a1a2-e5bcffa98cca/ascreenshot_9e255ba48f914c72ae57db7d3c1c7cd5_text_export.jpeg)
2. Click "Teams"
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/60a33a8c-2e48-4788-a1a2-e5bcffa98cca/ascreenshot_070934fa9c17453987f21f58117e673b_text_export.jpeg)
3. Click "+ Create New Team" (or edit an existing team)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/6f964ce2-f458-4719-a070-1af444ad92f5/ascreenshot_10f427f3106a4032a65d1046668880bd_text_export.jpeg)
4. Click "Router Settings"
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/a923c4ae-29f2-42b5-93ae-12f62d442691/ascreenshot_144520f2dd2f419dad79dffb1579ec04_text_export.jpeg)
5. Configure your desired router settings. For example, click "Fallbacks" to configure fallback models:
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/b062ecfa-bf5b-4c99-93a1-84b8b56fdb4c/ascreenshot_ea9acbc4e75448709b64a22addfb4157_text_export.jpeg)
6. Click "Select a model to begin configuring fallbacks" and configure your fallback chain:
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-24/67ca2655-4e82-4f93-be9a-7244ad22640f/ascreenshot_4fdbed826cd546d784e8738626be835d_text_export.jpeg)
## Use Cases
### Different Routing Strategies per Key
Configure different routing strategies for different use cases:
- **High-priority production keys**: Use `latency-based-routing` for optimal performance
- **Development keys**: Use `simple-shuffle` for simplicity
- **Cost-sensitive keys**: Use `cost-based-routing` to minimize expenses
### Team-Level Consistency
Apply consistent router settings across all keys in a team:
- Set team-wide fallback chains for reliability
- Configure team-specific timeout policies
- Apply uniform retry policies across team members
### Override Global Settings
Override global settings for specific scenarios:
- Production keys may need stricter timeout policies than development
- Certain teams may require different fallback models
- Individual keys may need custom retry policies for specific use cases
### Gradual Rollout
Test new router settings on specific keys or teams before applying globally:
- Configure new routing strategies on a test key first
- Validate fallback chains on a small team before global rollout
- A/B test different timeout values across different keys
## Related Features
- [Router Settings Reference](./config_settings.md#router_settings---reference) - Complete reference of all router settings
- [Load Balancing](./load_balancing.md) - Learn about routing strategies and load balancing
- [Reliability](./reliability.md) - Configure fallbacks, retries, and error handling
- [Keys](./keys.md) - Manage API keys and their settings
- [Teams](./teams.md) - Organize keys into teams

15
docs/my-website/docs/proxy/prometheus.md
View file

@ -121,8 +121,8 @@ Use this to track overall LiteLLM Proxy usage.
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "exception_status", "exception_class", "route"` |
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route"` |
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "user_email", "exception_status", "exception_class", "route", "model_id"` |
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"` |
### Callback Logging Metrics
@ -130,7 +130,12 @@ Monitor failures while shipping logs to downstream callbacks like `s3_v3` cold s
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_callback_logging_failures_metric` | Total number of failed attempts to emit logs to a configured callback. Labels: `"callback_name"`. Use this to alert on callback delivery issues such as repeated failures when writing to `s3_v3`. |
| `litellm_callback_logging_failures_metric` | Total number of failed attempts to emit logs to a configured callback. Labels: `"callback_name"`. Use this to alert on callback delivery issues such as repeated failures when writing to `s3_v3`, `langfuse`, or `langfuse_otel` and other otel providers |
**Supported Callbacks:**
- `S3Logger` - S3 v2 cold storage failures
- `langfuse` - Langfuse logging failures
- `otel` - OpenTelemetry logging failures
## LLM Provider Metrics
@ -191,10 +196,10 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model" |
| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model", "model_id" |
| `litellm_overhead_latency_metric` | Latency overhead (seconds) added by LiteLLM processing - tracked for labels "model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias" |
| `litellm_llm_api_latency_metric` | Latency (seconds) for just the LLM API call - tracked for labels "model", "hashed_api_key", "api_key_alias", "team", "team_alias", "requested_model", "end_user", "user" |
| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias` [Note: only emitted for streaming requests] |
| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias`, `requested_model`, `end_user`, `user`, `model_id` [Note: only emitted for streaming requests] |
## Tracking `end_user` on Prometheus

121
docs/my-website/docs/proxy/ui/page_visibility.md Normal file
View file

@ -0,0 +1,121 @@
import Image from '@theme/IdealImage';
# Control Page Visibility for Internal Users
Configure which navigation tabs and pages are visible to internal users (non-admin developers) in the LiteLLM UI.
Use this feature to simplify the UI and control which pages your internal users/developers can see when signing in.
## Overview
By default, all pages accessible to internal users are visible in the navigation sidebar. The page visibility control allows admins to restrict which pages internal users can see, creating a more focused and streamlined experience.
## Configure Page Visibility
### 1. Navigate to Settings
Click the **Settings** icon in the sidebar.
![Navigate to Settings](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/cbb6f272-ab18-4996-b57d-7ed4aad721ea/ascreenshot_ab80f3175b1a41b0bdabdd2cd3980573_text_export.jpeg)
### 2. Go to Admin Settings
Click **Admin Settings** from the settings menu.
![Go to Admin Settings](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/e2b327bf-1cfd-4519-a9ce-8a6ecb2de53a/ascreenshot_23bb1577b3f84d22be78e0faa58dee3d_text_export.jpeg)
### 3. Select UI Settings
Click **UI Settings** to access the page visibility controls.
![Select UI Settings](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/fff0366a-4944-457a-8f6a-e22018dde108/ascreenshot_0e268e8651654e75bb9fb40d2ed366a9_text_export.jpeg)
### 4. Open Page Visibility Configuration
Click **Configure Page Visibility** to expand the configuration panel.
![Open Configuration](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/3a4761d6-145a-4afd-8abf-d92744b9ac9f/ascreenshot_23c16eb79c32481887b879d961f1f00a_text_export.jpeg)
### 5. Select Pages to Make Visible
Check the boxes for the pages you want internal users to see. Pages are organized by category for easy navigation.
![Select Pages](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/b9c96b54-6c20-484f-8b0b-3a86decb5717/ascreenshot_3347ade01ebe4ea390bc7b57e53db43f_text_export.jpeg)
**Available pages include:**
- Virtual Keys
- Playground
- Models + Endpoints
- Agents
- MCP Servers
- Search Tools
- Vector Stores
- Logs
- Teams
- Organizations
- Usage
- Budgets
- And more...
### 6. Save Your Configuration
Click **Save Page Visibility Settings** to apply the changes.
![Save Settings](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/8a215378-44f5-4bb8-b984-06fa2aa03903/ascreenshot_44e7aeebe25a477ba92f73a3ed3df644_text_export.jpeg)
### 7. Verify Changes
Internal users will now only see the selected pages in their navigation sidebar.
![Verify Changes](https://colony-recorder.s3.amazonaws.com/files/2026-01-28/493a7718-b276-40b9-970f-5814054932d9/ascreenshot_ad23b8691f824095ba60256f91ad24f8_text_export.jpeg)
## Reset to Default
To restore all pages to internal users:
1. Open the Page Visibility configuration
2. Click **Reset to Default (All Pages)**
3. Click **Save Page Visibility Settings**
This will clear the restriction and show all accessible pages to internal users.
## API Configuration
You can also configure page visibility programmatically using the API:
### Get Current Settings
```bash
curl -X GET 'http://localhost:4000/ui_settings/get' \
-H 'Authorization: Bearer <your-admin-key>'
```
### Update Page Visibility
```bash
curl -X PATCH 'http://localhost:4000/ui_settings/update' \
-H 'Authorization: Bearer <your-admin-key>' \
-H 'Content-Type: application/json' \
-d '{
"enabled_ui_pages_internal_users": [
"api-keys",
"agents",
"mcp-servers",
"logs",
"teams"
]
}'
```
### Clear Page Visibility Restrictions
```bash
curl -X PATCH 'http://localhost:4000/ui_settings/update' \
-H 'Authorization: Bearer <your-admin-key>' \
-H 'Content-Type: application/json' \
-d '{
"enabled_ui_pages_internal_users": null
}'
```

78
docs/my-website/docs/rag_ingest.md
View file

@ -5,7 +5,7 @@ All-in-one document ingestion pipeline: **Upload → Chunk → Embed → Vector
| Feature | Supported |
|---------|-----------|
| Logging | Yes |
| Supported Providers | `openai`, `bedrock`, `vertex_ai`, `gemini` |
| Supported Providers | `openai`, `bedrock`, `vertex_ai`, `gemini`, `s3_vectors` |
:::tip
After ingesting documents, use [/rag/query](./rag_query.md) to search and generate responses with your ingested content.
@ -75,6 +75,31 @@ curl -X POST "http://localhost:4000/v1/rag/ingest" \
}"
```
### AWS S3 Vectors
```bash showLineNumbers title="Ingest to S3 Vectors"
curl -X POST "http://localhost:4000/v1/rag/ingest" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d "{
\"file\": {
\"filename\": \"document.txt\",
\"content\": \"$(base64 -i document.txt)\",
\"content_type\": \"text/plain\"
},
\"ingest_options\": {
\"embedding\": {
\"model\": \"text-embedding-3-small\"
},
\"vector_store\": {
\"custom_llm_provider\": \"s3_vectors\",
\"vector_bucket_name\": \"my-embeddings\",
\"aws_region_name\": \"us-west-2\"
}
}
}"
```
## Response
```json
@ -265,6 +290,57 @@ When `vector_store_id` is omitted, LiteLLM automatically creates:
4. Install: `pip install 'google-cloud-aiplatform>=1.60.0'`
:::
### vector_store (AWS S3 Vectors)
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `custom_llm_provider` | string | - | `"s3_vectors"` |
| `vector_bucket_name` | string | **required** | S3 vector bucket name |
| `index_name` | string | auto-create | Vector index name |
| `dimension` | integer | auto-detect | Vector dimension (auto-detected from embedding model) |
| `distance_metric` | string | `cosine` | Distance metric: `cosine` or `euclidean` |
| `non_filterable_metadata_keys` | array | `["source_text"]` | Metadata keys excluded from filtering |
| `aws_region_name` | string | `us-west-2` | AWS region |
| `aws_access_key_id` | string | env | AWS access key |
| `aws_secret_access_key` | string | env | AWS secret key |
:::info S3 Vectors Auto-Creation
When `index_name` is omitted, LiteLLM automatically creates:
- S3 vector bucket (if it doesn't exist)
- Vector index with auto-detected dimensions from your embedding model
**Dimension Auto-Detection**: The vector dimension is automatically detected by making a test embedding request to your specified model. No need to manually specify dimensions!
**Supported Embedding Models**: Works with any LiteLLM-supported embedding model (OpenAI, Cohere, Bedrock, Azure, etc.)
:::
**Example with auto-detection:**
```json
{
"embedding": {
"model": "text-embedding-3-small" // Dimension auto-detected as 1536
},
"vector_store": {
"custom_llm_provider": "s3_vectors",
"vector_bucket_name": "my-embeddings"
}
}
```
**Example with custom embedding provider:**
```json
{
"embedding": {
"model": "cohere/embed-english-v3.0" // Dimension auto-detected as 1024
},
"vector_store": {
"custom_llm_provider": "s3_vectors",
"vector_bucket_name": "my-embeddings",
"distance_metric": "cosine"
}
}
```
## Input Examples
### File (Base64)

6
docs/my-website/docs/routing.md
View file

@ -830,6 +830,12 @@ asyncio.run(router_acompletion())
</TabItem>
</Tabs>
## Traffic Mirroring / Silent Experiments
Traffic mirroring allows you to "mimic" production traffic to a secondary (silent) model for evaluation purposes. The silent model's response is gathered in the background and does not affect the latency or result of the primary request.
[**See detailed guide on A/B Testing - Traffic Mirroring here**](./traffic_mirroring.md)
## Basic Reliability
### Deployment Ordering (Priority)

83
docs/my-website/docs/traffic_mirroring.md Normal file
View file

@ -0,0 +1,83 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# A/B Testing - Traffic Mirroring
Traffic mirroring allows you to "mimic" production traffic to a secondary (silent) model for evaluation purposes. The silent model's response is gathered in the background and does not affect the latency or result of the primary request.
This is useful for:
- Testing a new model's performance on production prompts before switching.
- Comparing costs and latency between different providers.
- Debugging issues by mirroring traffic to a more verbose model.
## Quick Start
To enable traffic mirroring, add `silent_model` to the `litellm_params` of a deployment.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import Router
model_list = [
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/chatgpt-v-2",
"api_key": "...",
"silent_model": "gpt-4" # 👈 Mirror traffic to gpt-4
},
},
{
"model_name": "gpt-4",
"litellm_params": {
"model": "openai/gpt-4",
"api_key": "..."
},
}
]
router = Router(model_list=model_list)
# The request to "gpt-3.5-turbo" will trigger a background call to "gpt-4"
response = await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "How does traffic mirroring work?"}]
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
Add `silent_model` to your `config.yaml`:
```yaml
model_list:
- model_name: primary-model
litellm_params:
model: azure/gpt-35-turbo
api_key: os.environ/AZURE_API_KEY
silent_model: evaluation-model # 👈 Mirror traffic here
- model_name: evaluation-model
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
```
</TabItem>
</Tabs>
## How it works
1. **Request Received**: A request is made to a model group (e.g. `primary-model`).
2. **Deployment Picked**: LiteLLM picks a deployment from the group.
3. **Primary Call**: LiteLLM makes the call to the primary deployment.
4. **Mirroring**: If `silent_model` is present, LiteLLM triggers a background call to that model.
- For **Sync** calls: Uses a shared thread pool.
- For **Async** calls: Uses `asyncio.create_task`.
5. **Isolation**: The background call uses a `deepcopy` of the original request parameters and sets `metadata["is_silent_experiment"] = True`. It also strips out logging IDs to prevent collisions in usage tracking.
## Key Features
- **Latency Isolation**: The primary request returns as soon as it's ready. The background (silent) call does not block.
- **Unified Logging**: Background calls are processed via the Router, meaning they are automatically logged to your configured observability tools (Langfuse, S3, etc.).
- **Evaluation**: Use the `is_silent_experiment: True` flag in your logs to filter and compare results between the primary and mirrored calls.

115
docs/my-website/docs/tutorials/claude_agent_sdk.md Normal file
View file

@ -0,0 +1,115 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Claude Agent SDK with LiteLLM
Use Anthropic's Claude Agent SDK with any LLM provider through LiteLLM Proxy.
The Claude Agent SDK provides a high-level interface for building AI agents. By pointing it to LiteLLM, you can use the same agent code with OpenAI, Bedrock, Azure, Vertex AI, or any other provider.
## Quick Start
### 1. Install Dependencies
```bash
pip install claude-agent-sdk
```
### 2. Start LiteLLM Proxy
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: bedrock-claude-sonnet-3.5
litellm_params:
model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-opus-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-nova-premier
litellm_params:
model: "bedrock/amazon.nova-premier-v1:0"
aws_region_name: "us-east-1"
```
```bash
litellm --config config.yaml
```
### 3. Point Agent SDK to LiteLLM
| Environment Variable | Value | Description |
|---------------------|-------|-------------|
| `ANTHROPIC_BASE_URL` | `http://localhost:4000` | LiteLLM proxy URL |
| `ANTHROPIC_API_KEY` | `sk-1234` | Your LiteLLM API key (not Anthropic key) |
```python title="agent.py" showLineNumbers
import os
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
# Point to LiteLLM proxy (not Anthropic)
os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key
# Configure agent with any model from your config
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant.",
model="bedrock-claude-sonnet-4", # Use any model from config.yaml
max_turns=20,
)
async with ClaudeSDKClient(options=options) as client:
await client.query("What is LiteLLM?")
async for msg in client.receive_response():
if hasattr(msg, 'content'):
for content_block in msg.content:
if hasattr(content_block, 'text'):
print(content_block.text, end='', flush=True)
```
## Why Use LiteLLM with Agent SDK?
| Feature | Benefit |
|---------|---------|
| **Multi-Provider** | Use the same agent code with OpenAI, Bedrock, Azure, Vertex AI, etc. |
| **Cost Tracking** | Track spending across all agent conversations |
| **Rate Limiting** | Set budgets and limits on agent usage |
| **Load Balancing** | Distribute requests across multiple API keys or regions |
| **Fallbacks** | Automatically retry with different models if one fails |
## Complete Example
See our [cookbook example](https://github.com/BerriAI/litellm/tree/main/cookbook/anthropic_agent_sdk) for a complete interactive CLI agent that:
- Streams responses in real-time
- Switches between models dynamically
- Fetches available models from the proxy
```bash
# Clone and run the example
git clone https://github.com/BerriAI/litellm.git
cd litellm/cookbook/anthropic_agent_sdk
pip install -r requirements.txt
python main.py
```
## Related Resources
- [Claude Agent SDK Documentation](https://github.com/anthropics/anthropic-agent-sdk)
- [LiteLLM Proxy Quick Start](../proxy/quick_start)
- [Complete Cookbook Example](https://github.com/BerriAI/litellm/tree/main/cookbook/anthropic_agent_sdk)

BIN
docs/my-website/img/a2a_agent_spend.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 184 KiB

BIN
docs/my-website/img/a2a_trace_grouping.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 388 KiB

BIN
docs/my-website/img/ui_granular_router_settings.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 351 KiB

4
docs/my-website/release_notes/v1.81.0/index.md
View file

@ -1,5 +1,5 @@
---
title: "v1.81.0 - Claude Code - Web Search Across All Providers"
title: "v1.81.0-stable - Claude Code - Web Search Across All Providers"
slug: "v1-81-0"
date: 2026-01-18T10:00:00
authors:
@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.81.0.rc.1
docker.litellm.ai/berriai/litellm:v1.81.0-stable
```
</TabItem>

423
docs/my-website/release_notes/v1.81.3-stable/index.md Normal file
View file

@ -0,0 +1,423 @@
---
title: "v1.81.3-stable - Performance - 25% CPU Usage Reduction"
slug: "v1-81-3"
date: 2026-01-26T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.81.3.rc.2
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.81.3.rc.2
```
</TabItem>
</Tabs>
---
## New Models / Updated Models
### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Deprecation Date |
| -------- | ----- | -------------- | ------------------- | -------------------- | ---------------- |
| OpenAI | `gpt-audio`, `gpt-audio-2025-08-28` | 128K | $32/1M audio tokens, $2.5/1M text tokens | $64/1M audio tokens, $10/1M text tokens | - |
| OpenAI | `gpt-audio-mini`, `gpt-audio-mini-2025-08-28` | 128K | $10/1M audio tokens, $0.6/1M text tokens | $20/1M audio tokens, $2.4/1M text tokens | - |
| Deepinfra, Vertex AI, Google AI Studio, OpenRouter, Vercel AI Gateway | `gemini-2.0-flash-001`, `gemini-2.0-flash` | - | - | - | 2026-03-31 |
| Groq | `openai/gpt-oss-120b` | 131K | 0.075/1M cache read | 0.6/1M output tokens | - |
| Groq | `groq/openai/gpt-oss-20b` | 131K | 0.0375/1M cache read, $0.075/1M text tokens | 0.3/1M output tokens | - |
| Vertex AI | `gemini-2.5-computer-use-preview-10-2025` | 128K | $1.25 | $10 | - |
| Azure AI | `claude-haiku-4-5` | $1.25/1M cache read, $2/1M cache read above 1 hr, $0.1/1M text tokens | $5/1M output tokens | - |
| Azure AI | `claude-sonnet-4-5` | $3.75/1M cache read, $6/1M cache read above 1 hr, $3/1M text tokens | $15/1M output tokens | - |
| Azure AI | `claude-opus-4-5` | $6.25/1M cache read, $10/1M cache read above 1 hr, $0.5/1M text tokens | $25/1M output tokens | - |
| Azure AI | `claude-opus-4-1` | $18.75/1M cache read, $30/1M cache read above 1 hr, $1.5/1M text tokens | $75/1M output tokens | - |
### Features
- **[OpenAI](../../docs/providers/openai)**
- Add gpt-audio and gpt-audio-mini models to pricing - [PR #19509](https://github.com/BerriAI/litellm/pull/19509)
- correct audio token costs for gpt-4o-audio-preview models - [PR #19500](https://github.com/BerriAI/litellm/pull/19500)
- Limit stop sequence as per openai spec (ensures JetBrains IDE compatibility) - [PR #19562](https://github.com/BerriAI/litellm/pull/19562)
- **[VertexAI](../../docs/providers/vertex)**
- Docs - Google Workload Identity Federation (WIF) support - [PR #19320](https://github.com/BerriAI/litellm/pull/19320)
- **[Agentcore](../../docs/providers/bedrock_agentcore)**
- Fixes streaming issues with AWS Bedrock AgentCore where responses would stop after the first chunk, particularly affecting OAuth-enabled agents - [PR #17141](https://github.com/BerriAI/litellm/pull/17141)
- **[Chatgpt](../../docs/providers/chatgpt)**
- Adds support for calling chatgpt subscription via LiteLLM - [PR #19030](https://github.com/BerriAI/litellm/pull/19030)
- Adds responses API bridge support for chatgpt subscription provider - [PR #19030](https://github.com/BerriAI/litellm/pull/19030)
- **[Bedrock](../../docs/providers/bedrock)**
- support for output format for bedrock invoke via v1/messages - [PR #19560](https://github.com/BerriAI/litellm/pull/19560)
- **[Azure](../../docs/providers/azure/azure)**
- Add support for Azure OpenAI v1 API - [PR #19313](https://github.com/BerriAI/litellm/pull/19313)
- preserve content_policy_violation details for images (#19328) - [PR #19372](https://github.com/BerriAI/litellm/pull/19372)
- Support OpenAI-format nested tool definitions for Responses API - [PR #19526](https://github.com/BerriAI/litellm/pull/19526)
- **Gemini([Vertex AI](../../docs/providers/vertex), [Google AI Studio](../../docs/providers/gemini))**
- use responseJsonSchema for Gemini 2.0+ models - [PR #19314](https://github.com/BerriAI/litellm/pull/19314)
- **[Volcengine](../../docs/providers/volcano)**
- Support Volcengine responses api - [PR #18508](https://github.com/BerriAI/litellm/pull/18508)
- **[Anthropic](../../docs/providers/anthropic)**
- Add Support for calling Claude Code Max subscriptions via LiteLLM - [PR #19453](https://github.com/BerriAI/litellm/pull/19453)
- Add Structured output for /v1/messages with Anthropic API, Azure Anthropic API, Bedrock Converse - [PR #19545](https://github.com/BerriAI/litellm/pull/19545)
- **[Brave Search](../../docs/search/brave)**
- New Search provider - [PR #19433](https://github.com/BerriAI/litellm/pull/19433)
- **Sarvam ai**
- Add support for new sarvam models - [PR #19479](https://github.com/BerriAI/litellm/pull/19479)
- **[GMI](../../docs/providers/gmi)**
- add GMI Cloud provider support - [PR #19376](https://github.com/BerriAI/litellm/pull/19376)
### Bug Fixes
- **[Anthropic](../../docs/providers/anthropic)**
- Fix anthropic-beta sent client side being overridden instead of appended to - [PR #19343](https://github.com/BerriAI/litellm/pull/19343)
- Filter out unsupported fields from JSON schema for Anthropic's output_format API - [PR #19482](https://github.com/BerriAI/litellm/pull/19482)
- **[Bedrock](../../docs/providers/bedrock)**
- Expose stability models via /image_edits endpoint and ensure proper request transformation - [PR #19323](https://github.com/BerriAI/litellm/pull/19323)
- Claude Code x Bedrock Invoke fails with advanced-tool-use-2025-11-20 - [PR #19373](https://github.com/BerriAI/litellm/pull/19373)
- deduplicate tool calls in assistant history - [PR #19324](https://github.com/BerriAI/litellm/pull/19324)
- fix: correct us.anthropic.claude-opus-4-5 In-region pricing - [PR #19310](https://github.com/BerriAI/litellm/pull/19310)
- Fix request validation errors when using Claude 4 via bedrock invoke - [PR #19381](https://github.com/BerriAI/litellm/pull/19381)
- Handle thinking with tool calls for Claude 4 models - [PR #19506](https://github.com/BerriAI/litellm/pull/19506)
- correct streaming choice index for tool calls - [PR #19506](https://github.com/BerriAI/litellm/pull/19506)
- **[Ollama](../../docs/providers/ollama)**
- Fix tool call errors due with improved message extraction - [PR #19369](https://github.com/BerriAI/litellm/pull/19369)
- **[VertexAI](../../docs/providers/vertex)**
- Removed optional vertex_count_tokens_location param before request is sent to vertex - [PR #19359](https://github.com/BerriAI/litellm/pull/19359)
- **Gemini([Vertex AI](../../docs/providers/vertex), [Google AI Studio](../../docs/providers/gemini))**
- Supports setting media_resolution and fps parameters on each video file, when using Gemini video understanding - [PR #19273](https://github.com/BerriAI/litellm/pull/19273)
- handle reasoning_effort as dict from OpenAI Agents SDK - [PR #19419](https://github.com/BerriAI/litellm/pull/19419)
- add file content support in tool results - [PR #19416](https://github.com/BerriAI/litellm/pull/19416)
- **[Azure](../../docs/providers/azure_ai)**
- Fix Azure AI costs for Anthropic models - [PR #19530](https://github.com/BerriAI/litellm/pull/19530)
- **[Giga Chat](../../docs/providers/gigachat)**
- Add tool choice mapping - [PR #19645](https://github.com/BerriAI/litellm/pull/19645)
---
## AI API Endpoints (LLMs, MCP, Agents)
### Features
- **[Files API](../../docs/files_endpoints)**
- Add managed files support when load_balancing is True - [PR #19338](https://github.com/BerriAI/litellm/pull/19338)
- **[Claude Plugin Marketplace](../../docs/tutorials/claude_code_plugin_marketplace)**
- Add self hosted Claude Code Plugin Marketplace - [PR #19378](https://github.com/BerriAI/litellm/pull/19378)
- **[MCP](../../docs/mcp)**
- Add MCP Protocol version 2025-11-25 support - [PR #19379](https://github.com/BerriAI/litellm/pull/19379)
- Log MCP tool calls and list tools in the LiteLLM Spend Logs table for easier debugging - [PR #19469](https://github.com/BerriAI/litellm/pull/19469)
- **[Vertex AI](../../docs/providers/vertex)**
- Ensure only anthropic betas are forwarded down to LLM API (by default) - [PR #19542](https://github.com/BerriAI/litellm/pull/19542)
- Allow overriding to support forwarding incoming headers are forwarded down to target - [PR #19524](https://github.com/BerriAI/litellm/pull/19524)
- **[Chat/Completions](../../docs/completion/input)**
- Add MCP tools response to chat completions - [PR #19552](https://github.com/BerriAI/litellm/pull/19552)
- Add custom vertex ai finish reasons to the output - [PR #19558](https://github.com/BerriAI/litellm/pull/19558)
- Return MCP execution in /chat/completions before model output during streaming - [PR #19623](https://github.com/BerriAI/litellm/pull/19623)
### Bugs
- **[Responses API](../../docs/response_api)**
- Fix duplicate messages during MCP streaming tool execution - [PR #19317](https://github.com/BerriAI/litellm/pull/19317)
- Fix pickle error when using OpenAI's Responses API with stream=True and tool_choice of type allowed_tools (an OpenAI-native parameter) - [PR #17205](https://github.com/BerriAI/litellm/pull/17205)
- stream tool call events for non-openai models - [PR #19368](https://github.com/BerriAI/litellm/pull/19368)
- preserve tool output ordering for gemini in responses bridge - [PR #19360](https://github.com/BerriAI/litellm/pull/19360)
- Add ID caching to prevent ID mismatch text-start and text-delta - [PR #19390](https://github.com/BerriAI/litellm/pull/19390)
- Include output_item, reasoning_summary_Text_done and reasoning_summary_part_done events for non-openai models - [PR #19472](https://github.com/BerriAI/litellm/pull/19472)
- **[Chat/Completions](../../docs/completion/input)**
- fix: drop_params not dropping prompt_cache_key for non-OpenAI providers - [PR #19346](https://github.com/BerriAI/litellm/pull/19346)
- **[Realtime API](../../docs/realtime)**
- disable SSL for ws:// WebSocket connections - [PR #19345](https://github.com/BerriAI/litellm/pull/19345)
- **[Generate Content](../../docs/generateContent)**
- Log actual user input when google genai/vertex endpoints are called client-side - [PR #19156](https://github.com/BerriAI/litellm/pull/19156)
- **[/messages/count_tokens Anthropic Token Counting](../../docs/anthropic_count_tokens)**
- ensure it works for Anthropic, Azure AI Anthropic on AI Gateway - [PR #19432](https://github.com/BerriAI/litellm/pull/19432)
- **[MCP](../../docs/mcp)**
- forward static_headers to MCP servers - [PR #19366](https://github.com/BerriAI/litellm/pull/19366)
- **[Batch API](../../docs/batches)**
- Fix: generation config empty for batch - [PR #19556](https://github.com/BerriAI/litellm/pull/19556)
- **[Pass Through Endpoints](../../docs/proxy/pass_through)**
- Always reupdate registry - [PR #19420](https://github.com/BerriAI/litellm/pull/19420)
---
## Management Endpoints / UI
### Features
- **Cost Estimator**
- Fix model dropdown - [PR #19529](https://github.com/BerriAI/litellm/pull/19529)
- **Claude Code Plugins**
- Allow Adding Claude Code Plugins via UI - [PR #19387](https://github.com/BerriAI/litellm/pull/19387)
- **Guardrails**
- New Policy management UI - [PR #19668](https://github.com/BerriAI/litellm/pull/19668)
- Allow adding policies on Keys/Teams + Viewing on Info panels - [PR #19688](https://github.com/BerriAI/litellm/pull/19688)
- **General**
- respects custom authentication header override - [PR #19276](https://github.com/BerriAI/litellm/pull/19276)
- **Playground**
- Button to Fill Custom API Base - [PR #19440](https://github.com/BerriAI/litellm/pull/19440)
- display mcp output on the play ground - [PR #19553](https://github.com/BerriAI/litellm/pull/19553)
- **Models**
- Paginate /v2/models/info - [PR #19521](https://github.com/BerriAI/litellm/pull/19521)
- All Model Tab Pagination - [PR #19525](https://github.com/BerriAI/litellm/pull/19525)
- Adding Optional scope Param to /models - [PR #19539](https://github.com/BerriAI/litellm/pull/19539)
- Model Search - [PR #19622](https://github.com/BerriAI/litellm/pull/19622)
- Filter by Model ID and Team ID - [PR #19713](https://github.com/BerriAI/litellm/pull/19713)
- **MCP Servers**
- MCP Tools Tab Resetting to Overview - [PR #19468](https://github.com/BerriAI/litellm/pull/19468)
- **Organizations**
- Prevent org admin from creating a new user with proxy_admin permissions - [PR #19296](https://github.com/BerriAI/litellm/pull/19296)
- Edit Page: Reusable Model Select - [PR #19601](https://github.com/BerriAI/litellm/pull/19601)
- **Teams**
- Reusable Model Select - [PR #19543](https://github.com/BerriAI/litellm/pull/19543)
- [Fix] Team Update with Organization having All Proxy Models - [PR #19604](https://github.com/BerriAI/litellm/pull/19604)
- **Logs**
- Include tool arguments in spend logs table - [PR #19640](https://github.com/BerriAI/litellm/pull/19640)
- **Fallbacks / Loadbalancing**
- New fallbacks modal - [PR #19673](https://github.com/BerriAI/litellm/pull/19673)
- Set fallbacks/loadbalancing by team/key - [PR #19686](https://github.com/BerriAI/litellm/pull/19686)
### Bugs
- **Playground**
- increase model selector width in playground Compare view - [PR #19423](https://github.com/BerriAI/litellm/pull/19423)
- **Virtual Keys**
- Sorting Shows Incorrect Entries - [PR #19534](https://github.com/BerriAI/litellm/pull/19534)
- **General**
- UI 404 error when SERVER_ROOT_PATH is set - [PR #19467](https://github.com/BerriAI/litellm/pull/19467)
- Redirect to ui/login on expired JWT - [PR #19687](https://github.com/BerriAI/litellm/pull/19687)
- **SSO**
- Fix SSO user roles not updating for existing users - [PR #19621](https://github.com/BerriAI/litellm/pull/19621)
- **Guardrails**
- ensure guardrail patterns persist on edit and mode toggle - [PR #19265](https://github.com/BerriAI/litellm/pull/19265)
---
## AI Integrations
### Logging
- **General Logging**
- prevent printing duplicate StandardLoggingPayload logs - [PR #19325](https://github.com/BerriAI/litellm/pull/19325)
- Fix: log duplication when json_logs is enabled - [PR #19705](https://github.com/BerriAI/litellm/pull/19705)
- **Langfuse OTEL**
- ignore service logs and fix callback shadowing - [PR #19298](https://github.com/BerriAI/litellm/pull/19298)
- **Langfuse**
- Send litellm_trace_id - [PR #19528](https://github.com/BerriAI/litellm/pull/19528)
- Add Langfuse mock mode for testing without API calls - [PR #19676](https://github.com/BerriAI/litellm/pull/19676)
- **GCS Bucket**
- prevent unbounded queue growth due to slow API calls - [PR #19297](https://github.com/BerriAI/litellm/pull/19297)
- Add GCS mock mode for testing without API calls - [PR #19683](https://github.com/BerriAI/litellm/pull/19683)
- **Responses API Logging**
- Fix pydantic serialization error - [PR #19486](https://github.com/BerriAI/litellm/pull/19486)
- **Arize Phoenix**
- add openinference span kinds to arize phoenix - [PR #19267](https://github.com/BerriAI/litellm/pull/19267)
- **Prometheus**
- Added new prometheus metrics for user count and team count - [PR #19520](https://github.com/BerriAI/litellm/pull/19520)
### Guardrails
- **Bedrock Guardrails**
- Ensure post_call guardrail checks input+output - [PR #19151](https://github.com/BerriAI/litellm/pull/19151)
- **Prompt Security**
- fixing prompt-security's guardrail implementation - [PR #19374](https://github.com/BerriAI/litellm/pull/19374)
- **Presidio**
- Fixes crash in Presidio Guardrail when running in background threads (logging_hook) - [PR #19714](https://github.com/BerriAI/litellm/pull/19714)
- **Pillar Security**
- Migrate Pillar Security to Generic Guardrail API - [PR #19364](https://github.com/BerriAI/litellm/pull/19364)
- **Policy Engine**
- New LiteLLM Policy engine - create policies to manage guardrails, conditions - permissions per Key, Team - [PR #19612](https://github.com/BerriAI/litellm/pull/19612)
- **General**
- add case-insensitive support for guardrail mode and actions - [PR #19480](https://github.com/BerriAI/litellm/pull/19480)
### Prompt Management
- **General**
- fix prompt info lookup and delete using correct IDs - [PR #19358](https://github.com/BerriAI/litellm/pull/19358)
### Secret Manager
- **AWS Secret Manager**
- ensure auto-rotation updates existing AWS secret instead of creating new one - [PR #19455](https://github.com/BerriAI/litellm/pull/19455)
- **Hashicorp Vault**
- Ensure key rotations work with Vault - [PR #19634](https://github.com/BerriAI/litellm/pull/19634)
---
## Spend Tracking, Budgets and Rate Limiting
- **Pricing Updates**
- Add openai/dall-e base pricing entries - [PR #19133](https://github.com/BerriAI/litellm/pull/19133)
- Add `input_cost_per_video_per_second` in ModelInfoBase - [PR #19398](https://github.com/BerriAI/litellm/pull/19398)
---
## Performance / Loadbalancing / Reliability improvements
- **General**
- Fix date overflow/division by zero in proxy utils - [PR #19527](https://github.com/BerriAI/litellm/pull/19527)
- Fix in-flight request termination on SIGTERM when health-check runs in a separate process - [PR #19427](https://github.com/BerriAI/litellm/pull/19427)
- Fix Pass through routes to work with server root path - [PR #19383](https://github.com/BerriAI/litellm/pull/19383)
- Fix logging error for stop iteration - [PR #19649](https://github.com/BerriAI/litellm/pull/19649)
- prevent retrying 4xx client errors - [PR #19275](https://github.com/BerriAI/litellm/pull/19275)
- add better error handling for misconfig on health check - [PR #19441](https://github.com/BerriAI/litellm/pull/19441)
- **Router**
- Fix Azure RPM calculation formula - [PR #19513](https://github.com/BerriAI/litellm/pull/19513)
- Persist scheduler request queue to redis - [PR #19304](https://github.com/BerriAI/litellm/pull/19304)
- pass search_tools to Router during DB-triggered initialization - [PR #19388](https://github.com/BerriAI/litellm/pull/19388)
- Fixed PromptCachingCache to correctly handle messages where cache_control is a sibling key of string content - [PR #19266](https://github.com/BerriAI/litellm/pull/19266)
- **Memory Leaks/OOM**
- prevent OOM with nested $defs in tool schemas - [PR #19112](https://github.com/BerriAI/litellm/pull/19112)
- fix: HTTP client memory leaks in Presidio, OpenAI, and Gemini - [PR #19190](https://github.com/BerriAI/litellm/pull/19190)
- **Non root**
- fix logfile and pidfile of supervisor for non root environment - [PR #17267](https://github.com/BerriAI/litellm/pull/17267)
- resolve Read-only file system error in non-root images - [PR #19449](https://github.com/BerriAI/litellm/pull/19449)
- **Dockerfile**
- Redis Semantic Caching - add missing redisvl dependency to requirements.txt - [PR #19417](https://github.com/BerriAI/litellm/pull/19417)
- Bump OTEL versions to support a2a dependency - resolves modulenotfounderror for Microsoft Agents by @Harshit28j in #18991
- **DB**
- Handle PostgreSQL cached plan errors during rolling deployments - [PR #19424](https://github.com/BerriAI/litellm/pull/19424)
- **Timeouts**
- Fix: total timeout is not respected - [PR #19389](https://github.com/BerriAI/litellm/pull/19389)
- **SDK**
- Field-Existence Checks to Type Classes to Prevent Attribute Errors - [PR #18321](https://github.com/BerriAI/litellm/pull/18321)
- add google-cloud-aiplatform as optional dependency with clear error message - [PR #19437](https://github.com/BerriAI/litellm/pull/19437)
- Make grpc dependency optional - [PR #19447](https://github.com/BerriAI/litellm/pull/19447)
- Add support for retry policies - [PR #19645](https://github.com/BerriAI/litellm/pull/19645)
- **Performance**
- Cut chat_completion latency by ~21% by reducing pre-call processing time - [PR #19535](https://github.com/BerriAI/litellm/pull/19535)
- Optimize strip_trailing_slash with O(1) index check - [PR #19679](https://github.com/BerriAI/litellm/pull/19679)
- Optimize use_custom_pricing_for_model with set intersection - [PR #19677](https://github.com/BerriAI/litellm/pull/19677)
- perf: skip pattern_router.route() for non-wildcard models - [PR #19664](https://github.com/BerriAI/litellm/pull/19664)
- perf: Add LRU caching to get_model_info for faster cost lookups - [PR #19606](https://github.com/BerriAI/litellm/pull/19606)
---
## General Proxy Improvements
### Doc Improvements
- new tutorial for adding MCPs to Cursor via LiteLLM - [PR #19317](https://github.com/BerriAI/litellm/pull/19317)
- fix vertex_region to vertex_location in Vertex AI pass-through docs - [PR #19380](https://github.com/BerriAI/litellm/pull/19380)
- clarify Gemini and Vertex AI model prefix in json file - [PR #19443](https://github.com/BerriAI/litellm/pull/19443)
- update Claude Code integration guides - [PR #19415](https://github.com/BerriAI/litellm/pull/19415)
- adjust opencode tutorial - [PR #19605](https://github.com/BerriAI/litellm/pull/19605)
- add spend-queue-troubleshooting docs - [PR #19659](https://github.com/BerriAI/litellm/pull/19659)
- docs: add litellm-enterprise requirement for managed files - [PR #19689](https://github.com/BerriAI/litellm/pull/19689)
### Helm
- Add support for keda in helm chart - [PR #19337](https://github.com/BerriAI/litellm/pull/19337)
- sync Helm chart version with LiteLLM release version - [PR #19438](https://github.com/BerriAI/litellm/pull/19438)
- Enable PreStop hook configuration in values.yaml - [PR #19613](https://github.com/BerriAI/litellm/pull/19613)
### General
- Add health check scripts and parallel execution support - [PR #19295](https://github.com/BerriAI/litellm/pull/19295)
---
## New Contributors
* @dushyantzz made their first contribution in [PR #19158](https://github.com/BerriAI/litellm/pull/19158)
* @obod-mpw made their first contribution in [PR #19133](https://github.com/BerriAI/litellm/pull/19133)
* @msexxeta made their first contribution in [PR #19030](https://github.com/BerriAI/litellm/pull/19030)
* @rsicart made their first contribution in [PR #19337](https://github.com/BerriAI/litellm/pull/19337)
* @cluebbehusen made their first contribution in [PR #19311](https://github.com/BerriAI/litellm/pull/19311)
* @Lucky-Lodhi2004 made their first contribution in [PR #19315](https://github.com/BerriAI/litellm/pull/19315)
* @binbandit made their first contribution in [PR #19324](https://github.com/BerriAI/litellm/pull/19324)
* @flex-myeonghyeon made their first contribution in [PR #19381](https://github.com/BerriAI/litellm/pull/19381)
* @Lrakotoson made their first contribution in [PR #18321](https://github.com/BerriAI/litellm/pull/18321)
* @bensi94 made their first contribution in [PR #18787](https://github.com/BerriAI/litellm/pull/18787)
* @victorigualada made their first contribution in [PR #19368](https://github.com/BerriAI/litellm/pull/19368)
* @VedantMadane made their first contribution in #19266
* @stiyyagura0901 made their first contribution in #19276
* @kamilio made their first contribution in [PR #19447](https://github.com/BerriAI/litellm/pull/19447)
* @jonathansampson made their first contribution in [PR #19433](https://github.com/BerriAI/litellm/pull/19433)
* @rynecarbone made their first contribution in [PR #19416](https://github.com/BerriAI/litellm/pull/19416)
* @jayy-77 made their first contribution in #19366
* @davida-ps made their first contribution in [PR #19374](https://github.com/BerriAI/litellm/pull/19374)
* @joaodinissf made their first contribution in [PR #19506](https://github.com/BerriAI/litellm/pull/19506)
* @ecao310 made their first contribution in [PR #19520](https://github.com/BerriAI/litellm/pull/19520)
* @mpcusack-altos made their first contribution in [PR #19577](https://github.com/BerriAI/litellm/pull/19577)
* @milan-berri made their first contribution in [PR #19602](https://github.com/BerriAI/litellm/pull/19602)
* @xqe2011 made their first contribution in #19621
---
## Full Changelog
**[View complete changelog on GitHub](https://github.com/BerriAI/litellm/releases/tag/v1.81.3.rc)**

33
docs/my-website/sidebars.js
View file

@ -139,6 +139,20 @@ const sidebars = {
"tutorials/openai_codex"
]
},
{
type: "category",
label: "Agent SDKs",
link: {
type: "generated-index",
title: "Agent SDKs",
description: "Use LiteLLM with agent frameworks and SDKs",
slug: "/agent_sdks"
},
items: [
"tutorials/claude_agent_sdk",
"tutorials/google_adk",
]
},
],
// But you can create a sidebar manually
@ -274,11 +288,19 @@ const sidebars = {
"proxy/custom_sso",
"proxy/ai_hub",
"proxy/model_compare_ui",
"proxy/public_teams",
"proxy/self_serve",
"proxy/ui/bulk_edit_users",
"proxy/ui_credentials",
"tutorials/scim_litellm",
{
type: "category",
label: "UI User/Team Management",
items: [
"proxy/access_control",
"proxy/public_teams",
"proxy/self_serve",
"proxy/ui/bulk_edit_users",
"proxy/ui/page_visibility",
]
},
{
type: "category",
label: "UI Usage Tracking",
@ -364,6 +386,7 @@ const sidebars = {
label: "Load Balancing, Routing, Fallbacks",
href: "https://docs.litellm.ai/docs/routing-load-balancing",
},
"traffic_mirroring",
{
type: "category",
label: "Logging, Alerting, Metrics",
@ -775,6 +798,7 @@ const sidebars = {
"providers/oci",
"providers/ollama",
"providers/openrouter",
"providers/sarvam",
"providers/ovhcloud",
"providers/perplexity",
"providers/petals",
@ -843,6 +867,7 @@ const sidebars = {
"completion/image_generation_chat",
"completion/json_mode",
"completion/knowledgebase",
"providers/anthropic_tool_search",
"guides/code_interpreter",
"completion/message_trimming",
"completion/model_alias",
@ -879,6 +904,7 @@ const sidebars = {
"scheduler",
"proxy/auto_routing",
"proxy/load_balancing",
"proxy/keys_teams_router_settings",
"proxy/provider_budget_routing",
"proxy/reliability",
"proxy/fallback_management",
@ -919,7 +945,6 @@ const sidebars = {
type: "category",
label: "LiteLLM Python SDK Tutorials",
items: [
'tutorials/google_adk',
'tutorials/azure_openai',
'tutorials/instructor',
"tutorials/gradio_integration",

2
enterprise/litellm_enterprise/proxy/auth/route_checks.py
View file

@ -36,7 +36,7 @@ class EnterpriseRouteChecks:
if not premium_user:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"🚨🚨🚨 DISABLING LLM API ENDPOINTS is an Enterprise feature\n🚨 {CommonProxyErrors.not_premium_user.value}",
detail=f"🚨🚨🚨 DISABLING ADMIN ENDPOINTS is an Enterprise feature\n🚨 {CommonProxyErrors.not_premium_user.value}",
)
return get_secret_bool("DISABLE_ADMIN_ENDPOINTS") is True

87
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -244,6 +244,78 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return managed_object.created_by == user_id
return True # don't raise error if managed object is not found
async def list_user_batches(
self,
user_api_key_dict: UserAPIKeyAuth,
limit: Optional[int] = None,
after: Optional[str] = None,
provider: Optional[str] = None,
target_model_names: Optional[str] = None,
llm_router: Optional[Router] = None,
) -> Dict[str, Any]:
# Provider filtering is not supported for managed batches
# This is because the encoded object ids stored in the managed objects table do not contain the provider information
# To support provider filtering, we would need to store the provider information in the encoded object ids
if provider:
raise Exception(
"Filtering by 'provider' is not supported when using managed batches."
)
# Model name filtering is not supported for managed batches
# This is because the encoded object ids stored in the managed objects table do not contain the model name
# A hash of the model name + litellm_params for the model name is encoded as the model id. This is not sufficient to reliably map the target model names to the model ids.
if target_model_names:
raise Exception(
"Filtering by 'target_model_names' is not supported when using managed batches."
)
where_clause: Dict[str, Any] = {"file_purpose": "batch"}
# Filter by user who created the batch
if user_api_key_dict.user_id:
where_clause["created_by"] = user_api_key_dict.user_id
if after:
where_clause["id"] = {"gt": after}
# Fetch more than needed to allow for post-fetch filtering
fetch_limit = limit or 20
if target_model_names:
# Fetch extra to account for filtering
fetch_limit = max(fetch_limit * 3, 100)
batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
where=where_clause,
take=fetch_limit,
order={"created_at": "desc"},
)
batch_objects: List[LiteLLMBatch] = []
for batch in batches:
try:
# Stop once we have enough after filtering
if len(batch_objects) >= (limit or 20):
break
batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
batch_obj = LiteLLMBatch(**batch_data)
batch_obj.id = batch.unified_object_id
batch_objects.append(batch_obj)
except Exception as e:
verbose_logger.warning(
f"Failed to parse batch object {batch.unified_object_id}: {e}"
)
continue
return {
"object": "list",
"data": batch_objects,
"first_id": batch_objects[0].id if batch_objects else None,
"last_id": batch_objects[-1].id if batch_objects else None,
"has_more": len(batch_objects) == (limit or 20),
}
async def get_user_created_file_ids(
self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
) -> List[OpenAIFileObject]:
@ -297,6 +369,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if (
call_type == CallTypes.afile_content.value
or call_type == CallTypes.afile_delete.value
or call_type == CallTypes.afile_retrieve.value
or call_type == CallTypes.afile_content.value
):
await self.check_managed_file_id_access(data, user_api_key_dict)
@ -361,12 +435,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
data["model_file_id_mapping"] = model_file_id_mapping
elif (
call_type == CallTypes.aretrieve_batch.value
or call_type == CallTypes.acancel_batch.value
or call_type == CallTypes.acancel_fine_tuning_job.value
or call_type == CallTypes.aretrieve_fine_tuning_job.value
):
accessor_key: Optional[str] = None
retrieve_object_id: Optional[str] = None
if call_type == CallTypes.aretrieve_batch.value:
if (
call_type == CallTypes.aretrieve_batch.value
or call_type == CallTypes.acancel_batch.value
):
accessor_key = "batch_id"
elif (
call_type == CallTypes.acancel_fine_tuning_job.value
@ -382,6 +460,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if retrieve_object_id
else False
)
print(f"🔥potential_llm_object_id: {potential_llm_object_id}")
print(f"🔥retrieve_object_id: {retrieve_object_id}")
if potential_llm_object_id and retrieve_object_id:
## VALIDATE USER HAS ACCESS TO THE OBJECT ##
if not await self.can_user_call_unified_object_id(
@ -673,6 +753,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
bytes=file_objects[0].bytes,
filename=file_objects[0].filename,
status="uploaded",
expires_at=file_objects[0].expires_at,
)
return response
@ -893,8 +974,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
delete_response = None
specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
if specific_model_file_id_mapping:
# Remove conflicting keys from data to avoid duplicate keyword arguments
filtered_data = {k: v for k, v in data.items() if k not in ("model", "file_id")}
for model_id, model_file_id in specific_model_file_id_mapping.items():
delete_response = await llm_router.afile_delete(model=model_id, file_id=model_file_id, **data) # type: ignore
delete_response = await llm_router.afile_delete(model=model_id, file_id=model_file_id, **filtered_data) # type: ignore
stored_file_object = await self.delete_unified_file_id(
file_id, litellm_parent_otel_span

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.27-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.27.tar.gz vendored Normal file
View file

Binary file not shown.

40
litellm-proxy-extras/litellm_proxy_extras/_logging.py
View file

@ -1,12 +1,40 @@
import json
import logging
import os
from datetime import datetime
class JsonFormatter(logging.Formatter):
def formatTime(self, record, datefmt=None):
dt = datetime.fromtimestamp(record.created)
return dt.isoformat()
def format(self, record):
json_record = {
"message": record.getMessage(),
"level": record.levelname,
"timestamp": self.formatTime(record),
}
if record.exc_info:
json_record["stacktrace"] = self.formatException(record.exc_info)
return json.dumps(json_record)
def _is_json_enabled():
try:
import litellm
return getattr(litellm, 'json_logs', False)
except (ImportError, AttributeError):
return os.getenv("JSON_LOGS", "false").lower() == "true"
# Set up package logger
logger = logging.getLogger("litellm_proxy_extras")
if not logger.handlers: # Only add handler if none exists
if not logger.handlers:
handler = logging.StreamHandler()
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
handler.setFormatter(formatter)
if _is_json_enabled():
handler.setFormatter(JsonFormatter())
else:
handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
logger.addHandler(handler)
logger.setLevel(logging.INFO)

10
litellm-proxy-extras/litellm_proxy_extras/migrations/20251119131227_add_prompt_versioning/migration.sql
View file

@ -1,12 +1,12 @@
-- DropIndex
DROP INDEX "LiteLLM_PromptTable_prompt_id_key";
DROP INDEX IF EXISTS "LiteLLM_PromptTable_prompt_id_key";
-- AlterTable
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "version" INTEGER NOT NULL DEFAULT 1;
ALTER TABLE "LiteLLM_PromptTable"
ADD COLUMN "version" INTEGER NOT NULL DEFAULT 1;
-- CreateIndex
CREATE INDEX "LiteLLM_PromptTable_prompt_id_idx" ON "LiteLLM_PromptTable"("prompt_id");
CREATE INDEX "LiteLLM_PromptTable_prompt_id_idx" ON "LiteLLM_PromptTable" ("prompt_id");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_key" ON "LiteLLM_PromptTable"("prompt_id", "version");
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_key" ON "LiteLLM_PromptTable" ("prompt_id", "version");

5
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -760,6 +760,11 @@ model LiteLLM_ManagedVectorStoresTable {
updated_at DateTime @updatedAt
litellm_credential_name String?
litellm_params Json?
team_id String?
user_id String?
@@index([team_id])
@@index([user_id])
}
// Guardrails table for storing guardrail configurations

109
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -18,14 +18,15 @@ def str_to_bool(value: Optional[str]) -> bool:
return value.lower() in ("true", "1", "t", "y", "yes")
def _get_prisma_env() -> dict:
"""Get environment variables for Prisma, handling offline mode if configured."""
prisma_env = os.environ.copy()
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
# These env vars prevent Prisma from attempting downloads
prisma_env["NPM_CONFIG_PREFER_OFFLINE"] = "true"
prisma_env["NPM_CONFIG_CACHE"] = os.getenv("NPM_CONFIG_CACHE", "/app/.cache/npm")
prisma_env["NPM_CONFIG_CACHE"] = os.getenv(
"NPM_CONFIG_CACHE", "/app/.cache/npm"
)
return prisma_env
@ -34,29 +35,28 @@ def _get_prisma_command() -> str:
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
# Primary location where Prisma Python package installs the CLI
default_cli_path = "/app/.cache/prisma-python/binaries/node_modules/.bin/prisma"
# Check if custom path is provided (for flexibility)
custom_cli_path = os.getenv("PRISMA_CLI_PATH")
if custom_cli_path and os.path.exists(custom_cli_path):
logger.info(f"Using custom Prisma CLI at {custom_cli_path}")
return custom_cli_path
# Check the default location
if os.path.exists(default_cli_path):
logger.info(f"Using cached Prisma CLI at {default_cli_path}")
return default_cli_path
# If not found, log warning and fall back
logger.warning(
f"Prisma CLI not found at {default_cli_path}. "
"Falling back to Python wrapper (may attempt downloads)"
)
# Fall back to the Python wrapper (will work in online mode)
return "prisma"
class ProxyExtrasDBManager:
@staticmethod
def _get_prisma_dir() -> str:
@ -119,7 +119,7 @@ class ProxyExtrasDBManager:
stdout=open(migration_file, "w"),
check=True,
timeout=30,
env=prisma_env
env=prisma_env,
)
# 3. Mark the migration as applied since it represents current state
@ -134,7 +134,7 @@ class ProxyExtrasDBManager:
],
check=True,
timeout=30,
env=prisma_env
env=prisma_env,
)
return True
@ -159,14 +159,20 @@ class ProxyExtrasDBManager:
@staticmethod
def _roll_back_migration(migration_name: str):
"""Mark a specific migration as rolled back"""
# Set up environment for offline mode if configured
# Set up environment for offline mode if configured
prisma_env = _get_prisma_env()
subprocess.run(
[_get_prisma_command(), "migrate", "resolve", "--rolled-back", migration_name],
[
_get_prisma_command(),
"migrate",
"resolve",
"--rolled-back",
migration_name,
],
timeout=60,
check=True,
capture_output=True,
env=prisma_env
env=prisma_env,
)
@staticmethod
@ -178,7 +184,7 @@ class ProxyExtrasDBManager:
timeout=60,
check=True,
capture_output=True,
env=prisma_env
env=prisma_env,
)
@staticmethod
@ -228,6 +234,8 @@ class ProxyExtrasDBManager:
r"duplicate key value violates",
r"relation .* already exists",
r"constraint .* already exists",
r"does not exist",
r"Can't drop database.* because it doesn't exist",
]
for pattern in idempotent_patterns:
@ -248,7 +256,7 @@ class ProxyExtrasDBManager:
if not database_url:
logger.error("DATABASE_URL not set")
return
diff_dir = (
Path(migrations_dir)
/ "migrations"
@ -283,7 +291,7 @@ class ProxyExtrasDBManager:
check=True,
timeout=60,
stdout=f,
env=_get_prisma_env()
env=_get_prisma_env(),
)
except subprocess.CalledProcessError as e:
logger.warning(f"Failed to generate migration diff: {e.stderr}")
@ -313,7 +321,7 @@ class ProxyExtrasDBManager:
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
env=_get_prisma_env(),
)
logger.info(f"prisma db execute stdout: {result.stdout}")
logger.info("✅ Migration diff applied successfully")
@ -331,12 +339,18 @@ class ProxyExtrasDBManager:
try:
logger.info(f"Resolving migration: {migration_name}")
subprocess.run(
[_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
[
_get_prisma_command(),
"migrate",
"resolve",
"--applied",
migration_name,
],
timeout=60,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
env=_get_prisma_env(),
)
logger.debug(f"Resolved migration: {migration_name}")
except subprocess.CalledProcessError as e:
@ -375,7 +389,7 @@ class ProxyExtrasDBManager:
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
env=_get_prisma_env(),
)
logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@ -397,27 +411,42 @@ class ProxyExtrasDBManager:
)
if migration_match:
failed_migration = migration_match.group(1)
logger.info(
f"Found failed migration: {failed_migration}, marking as rolled back"
)
# Mark the failed migration as rolled back
subprocess.run(
[
_get_prisma_command(),
"migrate",
"resolve",
"--rolled-back",
failed_migration,
],
timeout=60,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
)
logger.info(
f"✅ Migration {failed_migration} marked as rolled back... retrying"
)
if ProxyExtrasDBManager._is_idempotent_error(e.stderr):
logger.info(
f"Migration {failed_migration} failed due to idempotent error (e.g., column already exists), resolving as applied"
)
ProxyExtrasDBManager._roll_back_migration(
failed_migration
)
ProxyExtrasDBManager._resolve_specific_migration(
failed_migration
)
logger.info(
f"✅ Migration {failed_migration} resolved."
)
return True
else:
logger.info(
f"Found failed migration: {failed_migration}, marking as rolled back"
)
# Mark the failed migration as rolled back
subprocess.run(
[
_get_prisma_command(),
"migrate",
"resolve",
"--rolled-back",
failed_migration,
],
timeout=60,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env(),
)
logger.info(
f"✅ Migration {failed_migration} marked as rolled back... retrying"
)
elif (
"P3005" in e.stderr
and "database schema is not empty" in e.stderr

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.26"
version = "0.4.27"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.26"
version = "0.4.27"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

4
litellm/__init__.py
View file

@ -80,6 +80,8 @@ import dotenv
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
if litellm_mode == "DEV":
dotenv.load_dotenv()
####################################################
if set_verbose:
_turn_on_debug()
@ -254,6 +256,7 @@ disable_streaming_logging: bool = False
disable_token_counter: bool = False
disable_add_transform_inline_image_block: bool = False
disable_add_user_agent_to_request_tags: bool = False
disable_anthropic_gemini_context_caching_transform: bool = False
extra_spend_tag_headers: Optional[List[str]] = None
in_memory_llm_clients_cache: "LLMClientCache"
safe_memory_mode: bool = False
@ -1467,6 +1470,7 @@ if TYPE_CHECKING:
from .llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config as AzureOpenAIGPT5Config
from .llms.azure.completion.transformation import AzureOpenAITextConfig as AzureOpenAITextConfig
from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig as HostedVLLMChatConfig
from .llms.hosted_vllm.embedding.transformation import HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig
from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig
from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig
from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig

1044
litellm/_lazy_imports_registry.py
View file

File diff suppressed because it is too large Load diff

60
litellm/_logging.py
View file

@ -166,6 +166,66 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
lg.propagate = False # prevent bubbling to parent/root
def _get_uvicorn_json_log_config():
"""
Generate a uvicorn log_config dictionary that applies JSON formatting to all loggers.
This ensures that uvicorn's access logs, error logs, and all application logs
are formatted as JSON when json_logs is enabled.
"""
json_formatter_class = "litellm._logging.JsonFormatter"
# Use the module-level log_level variable for consistency
uvicorn_log_level = log_level.upper()
log_config = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"json": {
"()": json_formatter_class,
},
"default": {
"()": json_formatter_class,
},
"access": {
"()": json_formatter_class,
},
},
"handlers": {
"default": {
"formatter": "json",
"class": "logging.StreamHandler",
"stream": "ext://sys.stdout",
},
"access": {
"formatter": "access",
"class": "logging.StreamHandler",
"stream": "ext://sys.stdout",
},
},
"loggers": {
"uvicorn": {
"handlers": ["default"],
"level": uvicorn_log_level,
"propagate": False,
},
"uvicorn.error": {
"handlers": ["default"],
"level": uvicorn_log_level,
"propagate": False,
},
"uvicorn.access": {
"handlers": ["access"],
"level": uvicorn_log_level,
"propagate": False,
},
},
}
return log_config
def _turn_on_json():
"""
Turn on JSON logging

97
litellm/a2a_protocol/card_resolver.py Normal file
View file

@ -0,0 +1,97 @@
"""
Custom A2A Card Resolver for LiteLLM.
Extends the A2A SDK's card resolver to support multiple well-known paths.
"""
from typing import TYPE_CHECKING, Any, Dict, Optional
from litellm._logging import verbose_logger
if TYPE_CHECKING:
from a2a.types import AgentCard
# Runtime imports with availability check
_A2ACardResolver: Any = None
AGENT_CARD_WELL_KNOWN_PATH: str = "/.well-known/agent-card.json"
PREV_AGENT_CARD_WELL_KNOWN_PATH: str = "/.well-known/agent.json"
try:
from a2a.client import A2ACardResolver as _A2ACardResolver # type: ignore[no-redef]
from a2a.utils.constants import ( # type: ignore[no-redef]
AGENT_CARD_WELL_KNOWN_PATH,
PREV_AGENT_CARD_WELL_KNOWN_PATH,
)
except ImportError:
pass
class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
"""
Custom A2A card resolver that supports multiple well-known paths.
Extends the base A2ACardResolver to try both:
- /.well-known/agent-card.json (standard)
- /.well-known/agent.json (previous/alternative)
"""
async def get_agent_card(
self,
relative_card_path: Optional[str] = None,
http_kwargs: Optional[Dict[str, Any]] = None,
) -> "AgentCard":
"""
Fetch the agent card, trying multiple well-known paths.
First tries the standard path, then falls back to the previous path.
Args:
relative_card_path: Optional path to the agent card endpoint.
If None, tries both well-known paths.
http_kwargs: Optional dictionary of keyword arguments to pass to httpx.get
Returns:
AgentCard from the A2A agent
Raises:
A2AClientHTTPError or A2AClientJSONError if both paths fail
"""
# If a specific path is provided, use the parent implementation
if relative_card_path is not None:
return await super().get_agent_card(
relative_card_path=relative_card_path,
http_kwargs=http_kwargs,
)
# Try both well-known paths
paths = [
AGENT_CARD_WELL_KNOWN_PATH,
PREV_AGENT_CARD_WELL_KNOWN_PATH,
]
last_error = None
for path in paths:
try:
verbose_logger.debug(
f"Attempting to fetch agent card from {self.base_url}{path}"
)
return await super().get_agent_card(
relative_card_path=path,
http_kwargs=http_kwargs,
)
except Exception as e:
verbose_logger.debug(
f"Failed to fetch agent card from {self.base_url}{path}: {e}"
)
last_error = e
continue
# If we get here, all paths failed - re-raise the last error
if last_error is not None:
raise last_error
# This shouldn't happen, but just in case
raise Exception(
f"Failed to fetch agent card from {self.base_url}. "
f"Tried paths: {', '.join(paths)}"
)

20
litellm/a2a_protocol/main.py
View file

@ -6,10 +6,11 @@ Provides standalone functions with @client decorator for LiteLLM logging integra
import asyncio
import datetime
import uuid
from typing import TYPE_CHECKING, Any, AsyncIterator, Coroutine, Dict, Optional, Union
import litellm
from litellm._logging import verbose_logger
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm.a2a_protocol.streaming_iterator import A2AStreamingIterator
from litellm.a2a_protocol.utils import A2ARequestUtils
from litellm.constants import DEFAULT_A2A_AGENT_TIMEOUT
@ -35,13 +36,18 @@ A2ACardResolver: Any = None
_A2AClient: Any = None
try:
from a2a.client import A2ACardResolver # type: ignore[no-redef]
from a2a.client import A2AClient as _A2AClient # type: ignore[no-redef]
A2A_SDK_AVAILABLE = True
except ImportError:
pass
# Import our custom card resolver that supports multiple well-known paths
from litellm.a2a_protocol.card_resolver import LiteLLMA2ACardResolver
# Use our custom resolver instead of the default A2A SDK resolver
A2ACardResolver = LiteLLMA2ACardResolver
def _set_usage_on_logging_obj(
kwargs: Dict[str, Any],
@ -225,7 +231,11 @@ async def asend_message(
raise ValueError(
"Either a2a_client or api_base is required for standard A2A flow"
)
a2a_client = await create_a2a_client(base_url=api_base)
trace_id = str(uuid.uuid4())
extra_headers = {"X-LiteLLM-Trace-Id": trace_id}
if agent_id:
extra_headers["X-LiteLLM-Agent-Id"] = agent_id
a2a_client = await create_a2a_client(base_url=api_base, extra_headers=extra_headers)
# Type assertion: a2a_client is guaranteed to be non-None here
assert a2a_client is not None
@ -490,6 +500,10 @@ async def create_a2a_client(
)
httpx_client = http_handler.client
if extra_headers:
httpx_client.headers.update(extra_headers)
verbose_proxy_logger.debug(f"A2A client created with extra_headers={extra_headers}")
# Resolve agent card
resolver = A2ACardResolver(
httpx_client=httpx_client,

14
litellm/batches/batch_utils.py
View file

@ -192,6 +192,9 @@ async def _get_batch_output_file_content_as_dictionary(
Get the batch output file content as a list of dictionaries
"""
from litellm.files.main import afile_content
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
)
if custom_llm_provider == "vertex_ai":
raise ValueError("Vertex AI does not support file content retrieval")
@ -199,8 +202,17 @@ async def _get_batch_output_file_content_as_dictionary(
if batch.output_file_id is None:
raise ValueError("Output file id is None cannot retrieve file content")
file_id = batch.output_file_id
is_base64_unified_file_id = _is_base64_encoded_unified_file_id(file_id)
if is_base64_unified_file_id:
try:
file_id = is_base64_unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
verbose_logger.debug(f"Extracted LLM output file ID from unified file ID: {file_id}")
except (IndexError, AttributeError) as e:
verbose_logger.error(f"Failed to extract LLM output file ID from unified file ID: {batch.output_file_id}, error: {e}")
_file_content = await afile_content(
file_id=batch.output_file_id,
file_id=file_id,
custom_llm_provider=custom_llm_provider,
)
return _get_file_content_as_dictionary(_file_content.content)

5
litellm/batches/main.py
View file

@ -31,7 +31,6 @@ from litellm.llms.openai.openai import OpenAIBatchesAPI
from litellm.llms.vertex_ai.batches.handler import VertexAIBatchPrediction
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
Batch,
CancelBatchRequest,
CreateBatchRequest,
RetrieveBatchRequest,
@ -868,7 +867,7 @@ async def acancel_batch(
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> Batch:
) -> LiteLLMBatch:
"""
Async: Cancels a batch.
@ -912,7 +911,7 @@ def cancel_batch(
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> Union[Batch, Coroutine[Any, Any, Batch]]:
) -> Union[LiteLLMBatch, Coroutine[Any, Any, LiteLLMBatch]]:
"""
Cancels a batch.

25
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -17,7 +17,7 @@ from typing import (
Optional,
Tuple,
Union,
cast,
cast
)
from openai.types.responses.tool_param import FunctionToolParam
@ -277,6 +277,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
responses_api_request["previous_response_id"] = value
elif key == "reasoning_effort":
responses_api_request["reasoning"] = self._map_reasoning_effort(value)
elif key == "web_search_options":
self._add_web_search_tool(responses_api_request, value)
# Get stream parameter from litellm_params if not in optional_params
stream = optional_params.get("stream") or litellm_params.get("stream", False)
@ -727,6 +729,27 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal")
return None
def _add_web_search_tool(
self,
responses_api_request: ResponsesAPIOptionalRequestParams,
web_search_options: Any,
) -> None:
"""
Add web search tool to responses API request.
Args:
responses_api_request: The responses API request dict to modify
web_search_options: Web search configuration (dict or other value)
"""
if "tools" not in responses_api_request or responses_api_request["tools"] is None:
responses_api_request["tools"] = []
web_search_tool: Dict[str, Any] = {"type": "web_search"}
if isinstance(web_search_options, dict):
web_search_tool.update(web_search_options)
responses_api_request["tools"].append(web_search_tool)
def _transform_response_format_to_text_format(
self, response_format: Union[Dict[str, Any], Any]
) -> Optional[Dict[str, Any]]:

14
litellm/constants.py
View file

@ -980,6 +980,7 @@ BEDROCK_CONVERSE_MODELS = [
"meta.llama3-2-90b-instruct-v1:0",
"amazon.nova-lite-v1:0",
"amazon.nova-2-lite-v1:0",
"amazon.nova-2-pro-preview-20251202-v1:0",
"amazon.nova-pro-v1:0",
"writer.palmyra-x4-v1:0",
"writer.palmyra-x5-v1:0",
@ -1165,6 +1166,12 @@ LITELLM_CLI_SOURCE_IDENTIFIER = "litellm-cli"
LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session"
CLI_JWT_TOKEN_NAME = "cli-jwt-token"
# Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility
CLI_JWT_EXPIRATION_HOURS = int(
os.getenv("CLI_JWT_EXPIRATION_HOURS")
or os.getenv("LITELLM_CLI_JWT_EXPIRATION_HOURS")
or 24
)
########################### DB CRON JOB NAMES ###########################
DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
@ -1326,6 +1333,13 @@ COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY = int(
DEFAULT_CHUNK_SIZE = int(os.getenv("DEFAULT_CHUNK_SIZE", 1000))
DEFAULT_CHUNK_OVERLAP = int(os.getenv("DEFAULT_CHUNK_OVERLAP", 200))
########################### S3 Vectors RAG Constants ###########################
S3_VECTORS_DEFAULT_DIMENSION = int(os.getenv("S3_VECTORS_DEFAULT_DIMENSION", 1024))
S3_VECTORS_DEFAULT_DISTANCE_METRIC = str(
os.getenv("S3_VECTORS_DEFAULT_DISTANCE_METRIC", "cosine")
)
S3_VECTORS_DEFAULT_NON_FILTERABLE_METADATA_KEYS = ["source_text"]
########################### Microsoft SSO Constants ###########################
MICROSOFT_USER_EMAIL_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_EMAIL_ATTRIBUTE", "userPrincipalName")

56
litellm/cost_calculator.py
View file

@ -23,7 +23,11 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import
from litellm.litellm_core_utils.llm_cost_calc.utils import (
CostCalculatorUtils,
_generic_cost_per_character,
_get_service_tier_cost_key,
_parse_prompt_tokens_details,
calculate_cost_component,
generic_cost_per_token,
get_billable_input_tokens,
select_cost_metric_for_model,
)
from litellm.llms.anthropic.cost_calculation import (
@ -431,12 +435,18 @@ def cost_per_token( # noqa: PLR0915
model=model, custom_llm_provider=custom_llm_provider
)
if model_info["input_cost_per_token"] > 0:
## COST PER TOKEN ##
prompt_tokens_cost_usd_dollar = (
model_info["input_cost_per_token"] * prompt_tokens
if (
model_info.get("input_cost_per_token", 0) > 0
or model_info.get("output_cost_per_token", 0) > 0
):
return generic_cost_per_token(
model=model,
usage=usage_block,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,
)
elif (
if (
model_info.get("input_cost_per_second", None) is not None
and response_time_ms is not None
):
@ -451,11 +461,7 @@ def cost_per_token( # noqa: PLR0915
model_info["input_cost_per_second"] * response_time_ms / 1000 # type: ignore
)
if model_info["output_cost_per_token"] > 0:
completion_tokens_cost_usd_dollar = (
model_info["output_cost_per_token"] * completion_tokens
)
elif (
if (
model_info.get("output_cost_per_second", None) is not None
and response_time_ms is not None
):
@ -955,7 +961,10 @@ def completion_cost( # noqa: PLR0915
router_model_id=router_model_id,
)
potential_model_names = [selected_model, _get_response_model(completion_response)]
potential_model_names = [
selected_model,
_get_response_model(completion_response),
]
if model is not None:
potential_model_names.append(model)
@ -1710,10 +1719,16 @@ def default_image_cost_calculator(
)
# Priority 1: Use per-image pricing if available (for gpt-image-1 and similar models)
if "input_cost_per_image" in cost_info and cost_info["input_cost_per_image"] is not None:
if (
"input_cost_per_image" in cost_info
and cost_info["input_cost_per_image"] is not None
):
return cost_info["input_cost_per_image"] * n
# Priority 2: Fall back to per-pixel pricing for backward compatibility
elif "input_cost_per_pixel" in cost_info and cost_info["input_cost_per_pixel"] is not None:
elif (
"input_cost_per_pixel" in cost_info
and cost_info["input_cost_per_pixel"] is not None
):
return cost_info["input_cost_per_pixel"] * height * width * n
else:
raise Exception(
@ -1833,9 +1848,22 @@ def batch_cost_calculator(
if input_cost_per_token_batches:
total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
elif input_cost_per_token:
# Subtract cached tokens from prompt_tokens before calculating cost
# Fixes issue where cached tokens are being charged again
total_prompt_cost = (
usage.prompt_tokens * (input_cost_per_token) / 2
get_billable_input_tokens(usage) * (input_cost_per_token) / 2
) # batch cost is usually half of the regular token cost
# Add cache read cost if applicable
details = _parse_prompt_tokens_details(usage)
cache_read_tokens = details["cache_hit_tokens"]
cache_read_cost_key = _get_service_tier_cost_key(
"cache_read_input_token_cost", None
)
total_prompt_cost += (
calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens)
/ 2
)
if output_cost_per_token_batches:
total_completion_cost = usage.completion_tokens * output_cost_per_token_batches
elif output_cost_per_token:

164
litellm/experimental_mcp_client/client.py
View file

@ -4,22 +4,26 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers.
import asyncio
import base64
from typing import Awaitable, Callable, Dict, List, Optional, TypeVar, Union
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, TypeVar, Union
import httpx
from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters
from mcp.client.sse import sse_client
from mcp.client.stdio import stdio_client
from mcp.client.streamable_http import streamable_http_client
try:
from mcp.client.streamable_http import streamable_http_client # type: ignore
except ImportError:
streamable_http_client = None
from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
from mcp.types import CallToolResult as MCPCallToolResult
from mcp.types import (
CallToolRequestParams as MCPCallToolRequestParams,
GetPromptRequestParams,
GetPromptResult,
Prompt,
ResourceTemplate,
TextContent,
)
from mcp.types import CallToolResult as MCPCallToolResult
from mcp.types import TextContent
from mcp.types import Tool as MCPTool
from pydantic import AnyUrl
@ -74,57 +78,91 @@ class MCPClient:
if auth_value:
self.update_auth_value(auth_value)
def _create_transport_context(
self,
) -> Tuple[Any, Optional[httpx.AsyncClient]]:
"""
Create the appropriate transport context based on transport type.
Returns:
Tuple of (transport_context, http_client).
http_client is only set for HTTP transport and needs cleanup.
"""
http_client: Optional[httpx.AsyncClient] = None
if self.transport_type == MCPTransport.stdio:
if not self.stdio_config:
raise ValueError("stdio_config is required for stdio transport")
server_params = StdioServerParameters(
command=self.stdio_config.get("command", ""),
args=self.stdio_config.get("args", []),
env=self.stdio_config.get("env", {}),
)
return stdio_client(server_params), None
if self.transport_type == MCPTransport.sse:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
return sse_client(
url=self.server_url,
timeout=self.timeout,
headers=headers,
httpx_client_factory=httpx_client_factory,
), None
# HTTP transport (default)
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
verbose_logger.debug(
"litellm headers for streamable_http_client: %s", headers
)
http_client = httpx_client_factory(
headers=headers,
timeout=httpx.Timeout(self.timeout),
)
transport_ctx = streamable_http_client(
url=self.server_url,
http_client=http_client,
)
return transport_ctx, http_client
async def _execute_session_operation(
self,
transport_ctx: Any,
operation: Callable[[ClientSession], Awaitable[TSessionResult]],
) -> TSessionResult:
"""
Execute an operation within a transport and session context.
Handles entering/exiting contexts and running the operation.
"""
transport = await transport_ctx.__aenter__()
try:
read_stream, write_stream = transport[0], transport[1]
session_ctx = ClientSession(read_stream, write_stream)
session = await session_ctx.__aenter__()
try:
await session.initialize()
return await operation(session)
finally:
try:
await session_ctx.__aexit__(None, None, None)
except BaseException as e:
verbose_logger.debug(f"Error during session context exit: {e}")
finally:
try:
await transport_ctx.__aexit__(None, None, None)
except BaseException as e:
verbose_logger.debug(f"Error during transport context exit: {e}")
async def run_with_session(
self, operation: Callable[[ClientSession], Awaitable[TSessionResult]]
) -> TSessionResult:
"""Open a session, run the provided coroutine, and clean up."""
transport_ctx = None
http_client: Optional[httpx.AsyncClient] = None
try:
if self.transport_type == MCPTransport.stdio:
if not self.stdio_config:
raise ValueError("stdio_config is required for stdio transport")
server_params = StdioServerParameters(
command=self.stdio_config.get("command", ""),
args=self.stdio_config.get("args", []),
env=self.stdio_config.get("env", {}),
)
transport_ctx = stdio_client(server_params)
elif self.transport_type == MCPTransport.sse:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
transport_ctx = sse_client(
url=self.server_url,
timeout=self.timeout,
headers=headers,
httpx_client_factory=httpx_client_factory,
)
else:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
verbose_logger.debug(
"litellm headers for streamable_http_client: %s", headers
)
http_client = httpx_client_factory(
headers=headers,
timeout=httpx.Timeout(self.timeout),
)
transport_ctx = streamable_http_client(
url=self.server_url,
http_client=http_client,
)
if transport_ctx is None:
raise RuntimeError("Failed to create transport context")
async with transport_ctx as transport:
read_stream, write_stream = transport[0], transport[1]
session_ctx = ClientSession(read_stream, write_stream)
async with session_ctx as session:
await session.initialize()
return await operation(session)
transport_ctx, http_client = self._create_transport_context()
return await self._execute_session_operation(transport_ctx, operation)
except Exception:
verbose_logger.warning(
"MCP client run_with_session failed for %s", self.server_url or "stdio"
@ -132,7 +170,10 @@ class MCPClient:
raise
finally:
if http_client is not None:
await http_client.aclose()
try:
await http_client.aclose()
except BaseException as e:
verbose_logger.debug(f"Error during http_client cleanup: {e}")
def update_auth_value(self, mcp_auth_value: Union[str, Dict[str, str]]):
"""
@ -245,7 +286,9 @@ class MCPClient:
return []
async def call_tool(
self, call_tool_request_params: MCPCallToolRequestParams
self,
call_tool_request_params: MCPCallToolRequestParams,
host_progress_callback: Optional[Callable] = None
) -> MCPCallToolResult:
"""
Call an MCP Tool.
@ -254,13 +297,28 @@ class MCPClient:
f"MCP client calling tool '{call_tool_request_params.name}' with arguments: {call_tool_request_params.arguments}"
)
async def on_progress(progress: float, total: float | None, message: str | None):
percentage = (progress / total * 100) if total else 0
verbose_logger.info(
f"MCP Tool '{call_tool_request_params.name}' progress: "
f"{progress}/{total} ({percentage:.0f}%) - {message or ''}"
)
# Forward to Host if callback provided
if host_progress_callback:
try:
await host_progress_callback(progress, total)
except Exception as e:
verbose_logger.warning(f"Failed to forward to Host: {e}")
async def _call_tool_operation(session: ClientSession):
verbose_logger.debug("MCP client sending tool call to session")
return await session.call_tool(
name=call_tool_request_params.name,
arguments=call_tool_request_params.arguments,
)
progress_callback=on_progress,
)
try:
tool_result = await self.run_with_session(_call_tool_operation)
verbose_logger.info(

13
litellm/files/main.py
View file

@ -9,6 +9,7 @@ import asyncio
import contextvars
import os
import time
import uuid
from functools import partial
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
@ -61,7 +62,7 @@ async def acreate_file(
file: FileTypes,
purpose: Literal["assistants", "batch", "fine-tune"],
expires_after: Optional[FileExpiresAfter] = None,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "manus"] = "openai",
custom_llm_provider: Literal["openai", "azure", "gemini", "vertex_ai", "bedrock", "hosted_vllm", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -106,7 +107,7 @@ def create_file(
file: FileTypes,
purpose: Literal["assistants", "batch", "fine-tune"],
expires_after: Optional[FileExpiresAfter] = None,
custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "manus"]] = None,
custom_llm_provider: Optional[Literal["openai", "azure", "gemini", "vertex_ai", "bedrock", "hosted_vllm", "manus"]] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -294,7 +295,7 @@ def create_file(
@client
async def afile_retrieve(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "manus"] = "openai",
custom_llm_provider: Literal["openai", "azure", "gemini", "hosted_vllm", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -493,7 +494,7 @@ def file_retrieve(
@client
async def afile_delete(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "manus"] = "openai",
custom_llm_provider: Literal["openai", "azure", "gemini", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -537,7 +538,7 @@ async def afile_delete(
def file_delete(
file_id: str,
model: Optional[str] = None,
custom_llm_provider: Union[Literal["openai", "azure", "manus"], str] = "openai",
custom_llm_provider: Union[Literal["openai", "azure", "gemini", "manus"], str] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -680,7 +681,7 @@ def file_delete(
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'file_delete'. Only 'openai', 'azure', and 'manus' are supported.".format(
message="LiteLLM doesn't support {} for 'file_delete'. Only 'openai', 'azure', 'gemini', and 'manus' are supported.".format(
custom_llm_provider
),
model="n/a",

29
litellm/integrations/callback_configs.json
View file

@ -83,6 +83,33 @@
},
"description": "Datadog Logging Integration"
},
{
"id": "datadog_cost_management",
"displayName": "Datadog Cost Management",
"logo": "datadog.png",
"supports_key_team_logging": false,
"dynamic_params": {
"dd_api_key": {
"type": "password",
"ui_name": "API Key",
"description": "Datadog API key for authentication",
"required": true
},
"dd_app_key": {
"type": "password",
"ui_name": "App Key",
"description": "Datadog Application Key for Cloud Cost Management",
"required": true
},
"dd_site": {
"type": "text",
"ui_name": "Site",
"description": "Datadog site URL (e.g., us5.datadoghq.com)",
"required": true
}
},
"description": "Datadog Cloud Cost Management Integration"
},
{
"id": "lago",
"displayName": "Lago",
@ -407,4 +434,4 @@
},
"description": "SQS Queue (AWS) Logging Integration"
}
]
]

16
litellm/integrations/custom_guardrail.py
View file

@ -516,7 +516,9 @@ class CustomGuardrail(CustomLogger):
from litellm.types.utils import GuardrailMode
# Use event_type if provided, otherwise fall back to self.event_hook
guardrail_mode: Union[GuardrailEventHooks, GuardrailMode, List[GuardrailEventHooks]]
guardrail_mode: Union[
GuardrailEventHooks, GuardrailMode, List[GuardrailEventHooks]
]
if event_type is not None:
guardrail_mode = event_type
elif isinstance(self.event_hook, Mode):
@ -524,11 +526,21 @@ class CustomGuardrail(CustomLogger):
else:
guardrail_mode = self.event_hook # type: ignore[assignment]
from litellm.litellm_core_utils.core_helpers import (
filter_exceptions_from_params,
)
# Sanitize the response to ensure it's JSON serializable and free of circular refs
# This prevents RecursionErrors in downstream loggers (Langfuse, Datadog, etc.)
clean_guardrail_response = filter_exceptions_from_params(
guardrail_json_response
)
slg = StandardLoggingGuardrailInformation(
guardrail_name=self.guardrail_name,
guardrail_provider=guardrail_provider,
guardrail_mode=guardrail_mode,
guardrail_response=guardrail_json_response,
guardrail_response=clean_guardrail_response,
guardrail_status=guardrail_status,
start_time=start_time,
end_time=end_time,

17
litellm/integrations/datadog/datadog.py
View file

@ -36,6 +36,7 @@ from litellm.integrations.datadog.datadog_handler import (
get_datadog_service,
get_datadog_source,
get_datadog_tags,
get_datadog_base_url_from_env,
)
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.llms.custom_httpx.http_handler import (
@ -110,7 +111,9 @@ class DataDogLogger(
self._configure_dd_direct_api()
# Optional override for testing
self._apply_dd_base_url_override()
dd_base_url = get_datadog_base_url_from_env()
if dd_base_url:
self.intake_url = f"{dd_base_url}/api/v2/logs"
self.sync_client = _get_httpx_client()
asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock()
@ -169,18 +172,6 @@ class DataDogLogger(
self.DD_API_KEY = os.getenv("DD_API_KEY")
self.intake_url = f"https://http-intake.logs.{os.getenv('DD_SITE')}/api/v2/logs"
def _apply_dd_base_url_override(self) -> None:
"""
Apply base URL override for testing purposes
"""
dd_base_url: Optional[str] = (
os.getenv("_DATADOG_BASE_URL")
or os.getenv("DATADOG_BASE_URL")
or os.getenv("DD_BASE_URL")
)
if dd_base_url is not None:
self.intake_url = f"{dd_base_url}/api/v2/logs"
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
"""
Async Log success events to Datadog

204
litellm/integrations/datadog/datadog_cost_management.py Normal file
View file

@ -0,0 +1,204 @@
import asyncio
import os
import time
from datetime import datetime
from typing import Dict, List, Optional, Tuple
from litellm._logging import verbose_logger
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.types.integrations.datadog_cost_management import (
DatadogFOCUSCostEntry,
)
from litellm.types.utils import StandardLoggingPayload
class DatadogCostManagementLogger(CustomBatchLogger):
def __init__(self, **kwargs):
self.dd_api_key = os.getenv("DD_API_KEY")
self.dd_app_key = os.getenv("DD_APP_KEY")
self.dd_site = os.getenv("DD_SITE", "datadoghq.com")
if not self.dd_api_key or not self.dd_app_key:
verbose_logger.warning(
"Datadog Cost Management: DD_API_KEY and DD_APP_KEY are required. Integration will not work."
)
self.upload_url = f"https://api.{self.dd_site}/api/v2/cost/custom_costs"
self.async_client = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
# Initialize lock and start periodic flush task
self.flush_lock = asyncio.Lock()
asyncio.create_task(self.periodic_flush())
# Check if flush_lock is already in kwargs to avoid double passing (unlikely but safe)
if "flush_lock" not in kwargs:
kwargs["flush_lock"] = self.flush_lock
super().__init__(**kwargs)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object", None
)
if standard_logging_object is None:
return
# Only log if there is a cost associated
if standard_logging_object.get("response_cost", 0) > 0:
self.log_queue.append(standard_logging_object)
if len(self.log_queue) >= self.batch_size:
await self.async_send_batch()
except Exception as e:
verbose_logger.exception(
f"Datadog Cost Management: Error in async_log_success_event: {str(e)}"
)
async def async_send_batch(self):
if not self.log_queue:
return
try:
# Aggregate costs from the batch
aggregated_entries = self._aggregate_costs(self.log_queue)
if not aggregated_entries:
return
# Send to Datadog
await self._upload_to_datadog(aggregated_entries)
# Clear queue only on success (or if we decide to drop on failure)
# CustomBatchLogger clears queue in flush_queue, so we just process here
except Exception as e:
verbose_logger.exception(
f"Datadog Cost Management: Error in async_send_batch: {str(e)}"
)
def _aggregate_costs(
self, logs: List[StandardLoggingPayload]
) -> List[DatadogFOCUSCostEntry]:
"""
Aggregates costs by Provider, Model, and Date.
Returns a list of DatadogFOCUSCostEntry.
"""
aggregator: Dict[Tuple[str, str, str, Tuple[Tuple[str, str], ...]], DatadogFOCUSCostEntry] = {}
for log in logs:
try:
# Extract keys for aggregation
provider = log.get("custom_llm_provider") or "unknown"
model = log.get("model") or "unknown"
cost = log.get("response_cost", 0)
if cost == 0:
continue
# Get date strings (FOCUS format requires specific keys, but for aggregation we group by Day)
# UTC date
# We interpret "ChargePeriod" as the day of the request.
ts = log.get("startTime") or time.time()
dt = datetime.fromtimestamp(ts)
date_str = dt.strftime("%Y-%m-%d")
# ChargePeriodStart and End
# If we want daily granularity, end date is usually same day or next day?
# Datadog Custom Costs usually expects periods.
# "ChargePeriodStart": "2023-01-01", "ChargePeriodEnd": "2023-12-31" in example.
# If we send daily, we can say Start=Date, End=Date.
# Grouping Key: Provider + Model + Date + Tags?
# For simplicity, let's aggregate by Provider + Model + Date first.
# If we handle tags, we need to include them in the key.
tags = self._extract_tags(log)
tags_key = tuple(sorted(tags.items())) if tags else ()
key = (provider, model, date_str, tags_key)
if key not in aggregator:
aggregator[key] = {
"ProviderName": provider,
"ChargeDescription": f"LLM Usage for {model}",
"ChargePeriodStart": date_str,
"ChargePeriodEnd": date_str,
"BilledCost": 0.0,
"BillingCurrency": "USD",
"Tags": tags if tags else None,
}
aggregator[key]["BilledCost"] += cost
except Exception as e:
verbose_logger.warning(
f"Error processing log for cost aggregation: {e}"
)
continue
return list(aggregator.values())
def _extract_tags(self, log: StandardLoggingPayload) -> Dict[str, str]:
from litellm.integrations.datadog.datadog_handler import (
get_datadog_env,
get_datadog_hostname,
get_datadog_pod_name,
get_datadog_service,
)
tags = {
"env": get_datadog_env(),
"service": get_datadog_service(),
"host": get_datadog_hostname(),
"pod_name": get_datadog_pod_name(),
}
# Add metadata as tags
metadata = log.get("metadata", {})
if metadata:
# Add user info
if "user_api_key_alias" in metadata:
tags["user"] = str(metadata["user_api_key_alias"])
if "user_api_key_team_alias" in metadata:
tags["team"] = str(metadata["user_api_key_team_alias"])
# model_group is not in StandardLoggingMetadata TypedDict, so we need to access it via dict.get()
model_group = metadata.get("model_group") # type: ignore[misc]
if model_group:
tags["model_group"] = str(model_group)
return tags
async def _upload_to_datadog(self, payload: List[Dict]):
if not self.dd_api_key or not self.dd_app_key:
return
headers = {
"Content-Type": "application/json",
"DD-API-KEY": self.dd_api_key,
"DD-APPLICATION-KEY": self.dd_app_key,
}
# The API endpoint expects a list of objects directly in the body (file content behavior)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
data_json = safe_dumps(payload)
response = await self.async_client.put(
self.upload_url, content=data_json, headers=headers
)
response.raise_for_status()
verbose_logger.debug(
f"Datadog Cost Management: Uploaded {len(payload)} cost entries. Status: {response.status_code}"
)

8
litellm/integrations/datadog/datadog_handler.py
View file

@ -20,6 +20,14 @@ def get_datadog_hostname() -> str:
return os.getenv("HOSTNAME", "")
def get_datadog_base_url_from_env() -> Optional[str]:
"""
Get base URL override from common DD_BASE_URL env var.
This is useful for testing or custom endpoints.
"""
return os.getenv("DD_BASE_URL")
def get_datadog_env() -> str:
return os.getenv("DD_ENV", "unknown")

58
litellm/integrations/datadog/datadog_llm_obs.py
View file

@ -25,6 +25,7 @@ from litellm.integrations.datadog.datadog_mock_client import (
from litellm.integrations.datadog.datadog_handler import (
get_datadog_service,
get_datadog_tags,
get_datadog_base_url_from_env,
)
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -60,18 +61,22 @@ class DataDogLLMObsLogger(CustomBatchLogger):
raise Exception(
"DD_SITE is not set, set 'DD_SITE=<>', example sit = `us5.datadoghq.com`"
)
# Configure DataDog endpoint (Agent or Direct API)
# Use LITELLM_DD_AGENT_HOST to avoid conflicts with ddtrace's DD_AGENT_HOST
dd_agent_host = os.getenv("LITELLM_DD_AGENT_HOST")
self.async_client = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
self.DD_API_KEY = os.getenv("DD_API_KEY")
self.DD_SITE = os.getenv("DD_SITE")
self.intake_url = (
f"https://api.{self.DD_SITE}/api/intake/llm-obs/v1/trace/spans"
)
# testing base url
dd_base_url = os.getenv("DD_BASE_URL")
if dd_agent_host:
self._configure_dd_agent(dd_agent_host=dd_agent_host)
else:
self._configure_dd_direct_api()
# Optional override for testing
dd_base_url = get_datadog_base_url_from_env()
if dd_base_url:
self.intake_url = f"{dd_base_url}/api/intake/llm-obs/v1/trace/spans"
@ -89,6 +94,38 @@ class DataDogLLMObsLogger(CustomBatchLogger):
verbose_logger.exception(f"DataDogLLMObs: Error initializing - {str(e)}")
raise e
def _configure_dd_agent(self, dd_agent_host: str):
"""
Configure the Datadog logger to send traces to the Agent.
"""
# When using the Agent, LLM Observability Intake does NOT require the API Key
# Reference: https://docs.datadoghq.com/llm_observability/setup/sdk/#agent-setup
# Use specific port for LLM Obs (Trace Agent) to avoid conflict with Logs Agent (10518)
agent_port = os.getenv("LITELLM_DD_LLM_OBS_PORT", "8126")
self.DD_SITE = "localhost" # Not used for URL construction in agent mode
self.intake_url = (
f"http://{dd_agent_host}:{agent_port}/api/intake/llm-obs/v1/trace/spans"
)
verbose_logger.debug(f"DataDogLLMObs: Using DD Agent at {self.intake_url}")
def _configure_dd_direct_api(self):
"""
Configure the Datadog logger to send traces directly to the Datadog API.
"""
if not self.DD_API_KEY:
raise Exception("DD_API_KEY is not set, set 'DD_API_KEY=<>'")
self.DD_SITE = os.getenv("DD_SITE")
if not self.DD_SITE:
raise Exception(
"DD_SITE is not set, set 'DD_SITE=<>', example site = `us5.datadoghq.com`"
)
self.intake_url = (
f"https://api.{self.DD_SITE}/api/intake/llm-obs/v1/trace/spans"
)
def _get_datadog_llm_obs_params(self) -> Dict:
"""
Get the datadog_llm_observability_params from litellm.datadog_llm_observability_params
@ -178,13 +215,14 @@ class DataDogLLMObsLogger(CustomBatchLogger):
json_payload = safe_dumps(payload)
headers = {"Content-Type": "application/json"}
if self.DD_API_KEY:
headers["DD-API-KEY"] = self.DD_API_KEY
response = await self.async_client.post(
url=self.intake_url,
content=json_payload,
headers={
"DD-API-KEY": self.DD_API_KEY,
"Content-Type": "application/json",
},
headers=headers,
)
if response.status_code != 202:

14
litellm/integrations/langfuse/langfuse.py
View file

@ -23,6 +23,7 @@ from litellm.constants import MAX_LANGFUSE_INITIALIZED_CLIENTS
from litellm.litellm_core_utils.core_helpers import (
safe_deep_copy,
reconstruct_model_name,
filter_exceptions_from_params,
)
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
from litellm.integrations.langfuse.langfuse_mock_client import (
@ -75,9 +76,8 @@ def _extract_cache_read_input_tokens(usage_obj) -> int:
# Check prompt_tokens_details.cached_tokens (used by Gemini and other providers)
if hasattr(usage_obj, "prompt_tokens_details"):
prompt_tokens_details = getattr(usage_obj, "prompt_tokens_details", None)
if (
prompt_tokens_details is not None
and hasattr(prompt_tokens_details, "cached_tokens")
if prompt_tokens_details is not None and hasattr(
prompt_tokens_details, "cached_tokens"
):
cached_tokens = getattr(prompt_tokens_details, "cached_tokens", None)
if (
@ -540,7 +540,6 @@ class LangFuseLogger:
verbose_logger.debug("Langfuse Layer Logging - logging to langfuse v2")
try:
metadata = metadata or {}
standard_logging_object: Optional[StandardLoggingPayload] = cast(
Optional[StandardLoggingPayload],
kwargs.get("standard_logging_object", None),
@ -706,9 +705,10 @@ class LangFuseLogger:
clean_metadata["litellm_response_cost"] = cost
if standard_logging_object is not None:
clean_metadata["hidden_params"] = standard_logging_object[
"hidden_params"
]
hidden_params = standard_logging_object.get("hidden_params", {})
clean_metadata["hidden_params"] = filter_exceptions_from_params(
hidden_params
)
if (
litellm.langfuse_default_tags is not None

92
litellm/integrations/langfuse/langfuse_prompt_management.py
View file

@ -300,43 +300,59 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
standard_callback_dynamic_params = kwargs.get(
"standard_callback_dynamic_params"
)
langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request(
globalLangfuseLogger=self,
standard_callback_dynamic_params=standard_callback_dynamic_params,
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
)
langfuse_logger_to_use.log_event_on_langfuse(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
user_id=kwargs.get("user", None),
)
try:
standard_callback_dynamic_params = kwargs.get(
"standard_callback_dynamic_params"
)
langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request(
globalLangfuseLogger=self,
standard_callback_dynamic_params=standard_callback_dynamic_params,
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
)
langfuse_logger_to_use.log_event_on_langfuse(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
user_id=kwargs.get("user", None),
)
except Exception as e:
from litellm._logging import verbose_logger
verbose_logger.exception(
f"Langfuse Layer Error - Exception occurred while logging success event: {str(e)}"
)
self.handle_callback_failure(callback_name="langfuse")
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
standard_callback_dynamic_params = kwargs.get(
"standard_callback_dynamic_params"
)
langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request(
globalLangfuseLogger=self,
standard_callback_dynamic_params=standard_callback_dynamic_params,
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
)
standard_logging_object = cast(
Optional[StandardLoggingPayload],
kwargs.get("standard_logging_object", None),
)
if standard_logging_object is None:
return
langfuse_logger_to_use.log_event_on_langfuse(
start_time=start_time,
end_time=end_time,
response_obj=None,
user_id=kwargs.get("user", None),
status_message=standard_logging_object["error_str"],
level="ERROR",
kwargs=kwargs,
)
try:
standard_callback_dynamic_params = kwargs.get(
"standard_callback_dynamic_params"
)
langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request(
globalLangfuseLogger=self,
standard_callback_dynamic_params=standard_callback_dynamic_params,
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
)
standard_logging_object = cast(
Optional[StandardLoggingPayload],
kwargs.get("standard_logging_object", None),
)
if standard_logging_object is None:
return
langfuse_logger_to_use.log_event_on_langfuse(
start_time=start_time,
end_time=end_time,
response_obj=None,
user_id=kwargs.get("user", None),
status_message=standard_logging_object["error_str"],
level="ERROR",
kwargs=kwargs,
)
except Exception as e:
from litellm._logging import verbose_logger
verbose_logger.exception(
f"Langfuse Layer Error - Exception occurred while logging failure event: {str(e)}"
)
self.handle_callback_failure(callback_name="langfuse")

18
litellm/integrations/opentelemetry.py
View file

@ -144,6 +144,7 @@ class OpenTelemetry(CustomLogger):
self.OTEL_EXPORTER = self.config.exporter
self.OTEL_ENDPOINT = self.config.endpoint
self.OTEL_HEADERS = self.config.headers
self._tracer_provider_cache: Dict[str, Any] = {}
self._init_tracing(tracer_provider)
_debug_otel = str(os.getenv("DEBUG_OTEL", "False")).lower()
@ -615,12 +616,20 @@ class OpenTelemetry(CustomLogger):
"""Create a temporary tracer with dynamic headers for this request only."""
from opentelemetry.sdk.trace import TracerProvider
# Prevents thread exhaustion by reusing providers for the same credential sets (e.g. per-team keys)
cache_key = str(sorted(dynamic_headers.items()))
if cache_key in self._tracer_provider_cache:
return self._tracer_provider_cache[cache_key].get_tracer(LITELLM_TRACER_NAME)
# Create a temporary tracer provider with dynamic headers
temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config))
temp_provider.add_span_processor(
self._get_span_processor(dynamic_headers=dynamic_headers)
)
# Store in cache for reuse
self._tracer_provider_cache[cache_key] = temp_provider
return temp_provider.get_tracer(LITELLM_TRACER_NAME)
def construct_dynamic_otel_headers(
@ -995,9 +1004,13 @@ class OpenTelemetry(CustomLogger):
from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider
try:
from opentelemetry.sdk._logs import LogRecord as SdkLogRecord # type: ignore[attr-defined] # OTEL < 1.39.0
from opentelemetry.sdk._logs import (
LogRecord as SdkLogRecord, # type: ignore[attr-defined] # OTEL < 1.39.0
)
except ImportError:
from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord # OTEL >= 1.39.0
from opentelemetry.sdk._logs._internal import (
LogRecord as SdkLogRecord, # OTEL >= 1.39.0
)
otel_logger = get_logger(LITELLM_LOGGER_NAME)
@ -1618,6 +1631,7 @@ class OpenTelemetry(CustomLogger):
)
except Exception as e:
self.handle_callback_failure(callback_name= self.callback_name)
verbose_logger.exception(
"OpenTelemetry logging error in set_attributes %s", str(e)
)

232
litellm/integrations/prometheus.py
View file

@ -229,14 +229,18 @@ class PrometheusLogger(CustomLogger):
self.litellm_remaining_api_key_requests_for_model = self._gauge_factory(
"litellm_remaining_api_key_requests_for_model",
"Remaining Requests API Key can make for model (model based rpm limit on key)",
labelnames=["hashed_api_key", "api_key_alias", "model"],
labelnames=self.get_labels_for_metric(
"litellm_remaining_api_key_requests_for_model"
),
)
# Remaining MODEL TPM limit for API Key
self.litellm_remaining_api_key_tokens_for_model = self._gauge_factory(
"litellm_remaining_api_key_tokens_for_model",
"Remaining Tokens API Key can make for model (model based tpm limit on key)",
labelnames=["hashed_api_key", "api_key_alias", "model"],
labelnames=self.get_labels_for_metric(
"litellm_remaining_api_key_tokens_for_model"
),
)
########################################
@ -312,6 +316,18 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric("litellm_deployment_state"),
)
self.litellm_deployment_tpm_limit = self._gauge_factory(
"litellm_deployment_tpm_limit",
"Deployment TPM limit found in config",
labelnames=self.get_labels_for_metric("litellm_deployment_tpm_limit"),
)
self.litellm_deployment_rpm_limit = self._gauge_factory(
"litellm_deployment_rpm_limit",
"Deployment RPM limit found in config",
labelnames=self.get_labels_for_metric("litellm_deployment_rpm_limit"),
)
self.litellm_deployment_cooled_down = self._counter_factory(
"litellm_deployment_cooled_down",
"LLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down",
@ -373,15 +389,9 @@ class PrometheusLogger(CustomLogger):
self.litellm_llm_api_failed_requests_metric = self._counter_factory(
name="litellm_llm_api_failed_requests_metric",
documentation="deprecated - use litellm_proxy_failed_requests_metric",
labelnames=[
"end_user",
"hashed_api_key",
"api_key_alias",
"model",
"team",
"team_alias",
"user",
],
labelnames=self.get_labels_for_metric(
"litellm_llm_api_failed_requests_metric"
),
)
self.litellm_requests_metric = self._counter_factory(
@ -891,7 +901,7 @@ class PrometheusLogger(CustomLogger):
model = kwargs.get("model", "")
litellm_params = kwargs.get("litellm_params", {}) or {}
_metadata = litellm_params.get("metadata", {})
_metadata = litellm_params.get("metadata") or {}
get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking()
end_user_id = get_end_user_id_for_cost_tracking(
@ -954,6 +964,8 @@ class PrometheusLogger(CustomLogger):
route=standard_logging_payload["metadata"].get(
"user_api_key_request_route"
),
client_ip=standard_logging_payload["metadata"].get("requester_ip_address"),
user_agent=standard_logging_payload["metadata"].get("user_agent"),
)
if (
@ -1011,6 +1023,7 @@ class PrometheusLogger(CustomLogger):
user_api_key_alias=user_api_key_alias,
kwargs=kwargs,
metadata=_metadata,
model_id=enum_values.model_id,
)
# set latency metrics
@ -1165,26 +1178,15 @@ class PrometheusLogger(CustomLogger):
response_cost: float,
user_id: Optional[str] = None,
):
_team_spend = litellm_params.get("metadata", {}).get(
"user_api_key_team_spend", None
)
_team_max_budget = litellm_params.get("metadata", {}).get(
"user_api_key_team_max_budget", None
)
_metadata = litellm_params.get("metadata") or {}
_team_spend = _metadata.get("user_api_key_team_spend", None)
_team_max_budget = _metadata.get("user_api_key_team_max_budget", None)
_api_key_spend = litellm_params.get("metadata", {}).get(
"user_api_key_spend", None
)
_api_key_max_budget = litellm_params.get("metadata", {}).get(
"user_api_key_max_budget", None
)
_api_key_spend = _metadata.get("user_api_key_spend", None)
_api_key_max_budget = _metadata.get("user_api_key_max_budget", None)
_user_spend = litellm_params.get("metadata", {}).get(
"user_api_key_user_spend", None
)
_user_max_budget = litellm_params.get("metadata", {}).get(
"user_api_key_user_max_budget", None
)
_user_spend = _metadata.get("user_api_key_user_spend", None)
_user_max_budget = _metadata.get("user_api_key_user_max_budget", None)
await self._set_api_key_budget_metrics_after_api_request(
user_api_key=user_api_key,
@ -1245,6 +1247,7 @@ class PrometheusLogger(CustomLogger):
user_api_key_alias: Optional[str],
kwargs: dict,
metadata: dict,
model_id: Optional[str] = None,
):
from litellm.proxy.common_utils.callback_utils import (
get_model_group_from_litellm_kwargs,
@ -1266,11 +1269,11 @@ class PrometheusLogger(CustomLogger):
)
self.litellm_remaining_api_key_requests_for_model.labels(
user_api_key, user_api_key_alias, model_group
user_api_key, user_api_key_alias, model_group, model_id
).set(remaining_requests)
self.litellm_remaining_api_key_tokens_for_model.labels(
user_api_key, user_api_key_alias, model_group
user_api_key, user_api_key_alias, model_group, model_id
).set(remaining_tokens)
def _set_latency_metrics(
@ -1296,12 +1299,14 @@ class PrometheusLogger(CustomLogger):
time_to_first_token_seconds is not None
and kwargs.get("stream", False) is True # only emit for streaming requests
):
_ttft_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_llm_api_time_to_first_token_metric"
),
enum_values=enum_values,
)
self.litellm_llm_api_time_to_first_token_metric.labels(
model,
user_api_key,
user_api_key_alias,
user_api_team,
user_api_team_alias,
**_ttft_labels
).observe(time_to_first_token_seconds)
else:
verbose_logger.debug(
@ -1341,7 +1346,7 @@ class PrometheusLogger(CustomLogger):
# request queue time (time from arrival to processing start)
_litellm_params = kwargs.get("litellm_params", {}) or {}
queue_time_seconds = _litellm_params.get("metadata", {}).get(
queue_time_seconds = (_litellm_params.get("metadata") or {}).get(
"queue_time_seconds"
)
if queue_time_seconds is not None and queue_time_seconds >= 0:
@ -1365,14 +1370,14 @@ class PrometheusLogger(CustomLogger):
standard_logging_payload: StandardLoggingPayload = kwargs.get(
"standard_logging_object", {}
)
if self._should_skip_metrics_for_invalid_key(
kwargs=kwargs, standard_logging_payload=standard_logging_payload
):
return
model = kwargs.get("model", "")
litellm_params = kwargs.get("litellm_params", {}) or {}
get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking()
@ -1396,6 +1401,7 @@ class PrometheusLogger(CustomLogger):
user_api_team,
user_api_team_alias,
user_id,
standard_logging_payload.get("model_id", ""),
).inc()
self.set_llm_deployment_failure_metrics(kwargs)
except Exception as e:
@ -1413,49 +1419,57 @@ class PrometheusLogger(CustomLogger):
) -> Optional[int]:
"""
Extract HTTP status code from various input formats for validation.
This is a centralized helper to extract status code from different
callback function signatures. Handles both ProxyException (uses 'code')
and standard exceptions (uses 'status_code').
Args:
kwargs: Dictionary potentially containing 'exception' key
enum_values: Object with 'status_code' attribute
exception: Exception object to extract status code from directly
Returns:
Status code as integer if found, None otherwise
"""
status_code = None
# Try from enum_values first (most common in our callbacks)
if enum_values and hasattr(enum_values, "status_code") and enum_values.status_code:
if (
enum_values
and hasattr(enum_values, "status_code")
and enum_values.status_code
):
try:
status_code = int(enum_values.status_code)
except (ValueError, TypeError):
pass
if not status_code and exception:
# ProxyException uses 'code' attribute, other exceptions may use 'status_code'
status_code = getattr(exception, "status_code", None) or getattr(exception, "code", None)
status_code = getattr(exception, "status_code", None) or getattr(
exception, "code", None
)
if status_code is not None:
try:
status_code = int(status_code)
except (ValueError, TypeError):
status_code = None
if not status_code and kwargs:
exception_in_kwargs = kwargs.get("exception")
if exception_in_kwargs:
status_code = getattr(exception_in_kwargs, "status_code", None) or getattr(exception_in_kwargs, "code", None)
status_code = getattr(
exception_in_kwargs, "status_code", None
) or getattr(exception_in_kwargs, "code", None)
if status_code is not None:
try:
status_code = int(status_code)
except (ValueError, TypeError):
status_code = None
return status_code
def _is_invalid_api_key_request(
self,
status_code: Optional[int],
@ -1463,23 +1477,23 @@ class PrometheusLogger(CustomLogger):
) -> bool:
"""
Determine if a request has an invalid API key based on status code and exception.
This method prevents invalid authentication attempts from being recorded in
Prometheus metrics. A 401 status code is the definitive indicator of authentication
failure. Additionally, we check exception messages for authentication error patterns
to catch cases where the exception hasn't been converted to a ProxyException yet.
Args:
status_code: HTTP status code (401 indicates authentication error)
exception: Exception object to check for auth-related error messages
Returns:
True if the request has an invalid API key and metrics should be skipped,
False otherwise
"""
if status_code == 401:
return True
# Handle cases where AssertionError is raised before conversion to ProxyException
if exception is not None:
exception_str = str(exception).lower()
@ -1492,9 +1506,9 @@ class PrometheusLogger(CustomLogger):
]
if any(pattern in exception_str for pattern in auth_error_patterns):
return True
return False
def _should_skip_metrics_for_invalid_key(
self,
kwargs: Optional[dict] = None,
@ -1505,18 +1519,18 @@ class PrometheusLogger(CustomLogger):
) -> bool:
"""
Determine if Prometheus metrics should be skipped for invalid API key requests.
This is a centralized validation method that extracts status code and exception
information from various callback function signatures and determines if the request
represents an invalid API key attempt that should be filtered from metrics.
Args:
kwargs: Dictionary potentially containing exception and other data
user_api_key_dict: User API key authentication object (currently unused)
enum_values: Object with status_code attribute
standard_logging_payload: Standard logging payload dictionary
exception: Exception object to check directly
Returns:
True if metrics should be skipped (invalid key detected), False otherwise
"""
@ -1525,17 +1539,17 @@ class PrometheusLogger(CustomLogger):
enum_values=enum_values,
exception=exception,
)
if exception is None and kwargs:
exception = kwargs.get("exception")
if self._is_invalid_api_key_request(status_code, exception=exception):
verbose_logger.debug(
"Skipping Prometheus metrics for invalid API key request: "
f"status_code={status_code}, exception={type(exception).__name__ if exception else None}"
)
return True
return False
async def async_post_call_failure_hook(
@ -1576,6 +1590,10 @@ class PrometheusLogger(CustomLogger):
litellm_params=request_data,
proxy_server_request=request_data.get("proxy_server_request", {}),
)
_metadata = request_data.get("metadata", {}) or {}
model_id = _metadata.get("model_info", {}).get("id") or request_data.get(
"model_info", {}
).get("id")
enum_values = UserAPIKeyLabelValues(
end_user=user_api_key_dict.end_user_id,
user=user_api_key_dict.user_id,
@ -1590,6 +1608,9 @@ class PrometheusLogger(CustomLogger):
exception_class=self._get_exception_class_name(original_exception),
tags=_tags,
route=user_api_key_dict.request_route,
client_ip=_metadata.get("requester_ip_address"),
user_agent=_metadata.get("user_agent"),
model_id=model_id,
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
@ -1629,6 +1650,7 @@ class PrometheusLogger(CustomLogger):
):
return
_metadata = data.get("metadata", {}) or {}
enum_values = UserAPIKeyLabelValues(
end_user=user_api_key_dict.end_user_id,
hashed_api_key=user_api_key_dict.api_key,
@ -1644,6 +1666,8 @@ class PrometheusLogger(CustomLogger):
litellm_params=data,
proxy_server_request=data.get("proxy_server_request", {}),
),
client_ip=_metadata.get("requester_ip_address"),
user_agent=_metadata.get("user_agent"),
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
@ -1684,7 +1708,7 @@ class PrometheusLogger(CustomLogger):
exception = request_kwargs.get("exception", None)
llm_provider = _litellm_params.get("custom_llm_provider", None)
if self._should_skip_metrics_for_invalid_key(
kwargs=request_kwargs,
standard_logging_payload=standard_logging_payload,
@ -1716,6 +1740,10 @@ class PrometheusLogger(CustomLogger):
"user_api_key_team_alias"
],
tags=standard_logging_payload.get("request_tags", []),
client_ip=standard_logging_payload["metadata"].get(
"requester_ip_address"
),
user_agent=standard_logging_payload["metadata"].get("user_agent"),
)
"""
@ -1753,6 +1781,49 @@ class PrometheusLogger(CustomLogger):
)
)
def _set_deployment_tpm_rpm_limit_metrics(
self,
model_info: dict,
litellm_params: dict,
litellm_model_name: Optional[str],
model_id: Optional[str],
api_base: Optional[str],
llm_provider: Optional[str],
):
"""
Set the deployment TPM and RPM limits metrics
"""
tpm = model_info.get("tpm") or litellm_params.get("tpm")
rpm = model_info.get("rpm") or litellm_params.get("rpm")
if tpm is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_deployment_tpm_limit"
),
enum_values=UserAPIKeyLabelValues(
litellm_model_name=litellm_model_name,
model_id=model_id,
api_base=api_base,
api_provider=llm_provider,
),
)
self.litellm_deployment_tpm_limit.labels(**_labels).set(tpm)
if rpm is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_deployment_rpm_limit"
),
enum_values=UserAPIKeyLabelValues(
litellm_model_name=litellm_model_name,
model_id=model_id,
api_base=api_base,
api_provider=llm_provider,
),
)
self.litellm_deployment_rpm_limit.labels(**_labels).set(rpm)
def set_llm_deployment_success_metrics(
self,
request_kwargs: dict,
@ -1786,6 +1857,16 @@ class PrometheusLogger(CustomLogger):
_model_info = _metadata.get("model_info") or {}
model_id = _model_info.get("id", None)
if _model_info or _litellm_params:
self._set_deployment_tpm_rpm_limit_metrics(
model_info=_model_info,
litellm_params=_litellm_params,
litellm_model_name=litellm_model_name,
model_id=model_id,
api_base=api_base,
llm_provider=llm_provider,
)
remaining_requests: Optional[int] = None
remaining_tokens: Optional[int] = None
if additional_headers := standard_logging_payload["hidden_params"][
@ -2263,7 +2344,10 @@ class PrometheusLogger(CustomLogger):
async def fetch_keys(
page_size: int, page: int
) -> Tuple[List[Union[str, UserAPIKeyAuth, LiteLLM_DeletedVerificationToken]], Optional[int]]:
) -> Tuple[
List[Union[str, UserAPIKeyAuth, LiteLLM_DeletedVerificationToken]],
Optional[int],
]:
key_list_response = await _list_key_helper(
prisma_client=prisma_client,
page=page,
@ -2379,12 +2463,16 @@ class PrometheusLogger(CustomLogger):
# Get total user count
total_users = await prisma_client.db.litellm_usertable.count()
self.litellm_total_users_metric.set(total_users)
verbose_logger.debug(f"Prometheus: set litellm_total_users to {total_users}")
verbose_logger.debug(
f"Prometheus: set litellm_total_users to {total_users}"
)
# Get total team count
total_teams = await prisma_client.db.litellm_teamtable.count()
self.litellm_teams_count_metric.set(total_teams)
verbose_logger.debug(f"Prometheus: set litellm_teams_count to {total_teams}")
verbose_logger.debug(
f"Prometheus: set litellm_teams_count to {total_teams}"
)
except Exception as e:
verbose_logger.exception(
f"Error initializing user/team count metrics: {str(e)}"
@ -2412,8 +2500,8 @@ class PrometheusLogger(CustomLogger):
self,
user_api_team: Optional[str],
user_api_team_alias: Optional[str],
team_spend: float,
team_max_budget: float,
team_spend: Optional[float],
team_max_budget: Optional[float],
response_cost: float,
):
"""
@ -2575,7 +2663,7 @@ class PrometheusLogger(CustomLogger):
user_api_key: Optional[str],
user_api_key_alias: Optional[str],
response_cost: float,
key_max_budget: float,
key_max_budget: Optional[float],
key_spend: Optional[float],
):
if user_api_key:
@ -2592,7 +2680,7 @@ class PrometheusLogger(CustomLogger):
self,
user_api_key: str,
user_api_key_alias: str,
key_max_budget: float,
key_max_budget: Optional[float],
key_spend: Optional[float],
response_cost: float,
) -> UserAPIKeyAuth:

4
litellm/litellm_core_utils/core_helpers.py
View file

@ -351,9 +351,9 @@ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
# Skip callable objects (functions, methods, lambdas) but not classes (type objects)
if callable(data) and not isinstance(data, type):
return None
# Skip known non-serializable object types (Logging, etc.)
# Skip known non-serializable object types (Logging, Router, etc.)
obj_type_name = type(data).__name__
if obj_type_name in ["Logging", "LiteLLMLoggingObj"]:
if obj_type_name in ["Logging", "LiteLLMLoggingObj", "Router"]:
return None
if isinstance(data, dict):

9
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -93,8 +93,11 @@ def get_litellm_params(
"text_completion": text_completion,
"azure_ad_token_provider": azure_ad_token_provider,
"user_continue_message": user_continue_message,
"base_model": base_model or (
_get_base_model_from_litellm_call_metadata(metadata=metadata) if metadata else None
"base_model": base_model
or (
_get_base_model_from_litellm_call_metadata(metadata=metadata)
if metadata
else None
),
"litellm_trace_id": litellm_trace_id,
"litellm_session_id": litellm_session_id,
@ -139,5 +142,7 @@ def get_litellm_params(
"aws_sts_endpoint": kwargs.get("aws_sts_endpoint"),
"aws_external_id": kwargs.get("aws_external_id"),
"aws_bedrock_runtime_endpoint": kwargs.get("aws_bedrock_runtime_endpoint"),
"tpm": kwargs.get("tpm"),
"rpm": kwargs.get("rpm"),
}
return litellm_params

52
litellm/litellm_core_utils/litellm_logging.py
View file

@ -335,7 +335,9 @@ class Logging(LiteLLMLoggingBaseClass):
self.start_time = start_time # log the call start time
self.call_type = call_type
self.litellm_call_id = litellm_call_id
self.litellm_trace_id: str = litellm_trace_id if litellm_trace_id else str(uuid.uuid4())
self.litellm_trace_id: str = (
litellm_trace_id if litellm_trace_id else str(uuid.uuid4())
)
self.function_id = function_id
self.streaming_chunks: List[Any] = [] # for generating complete stream response
self.sync_streaming_chunks: List[
@ -544,7 +546,10 @@ class Logging(LiteLLMLoggingBaseClass):
if "stream_options" in additional_params:
self.stream_options = additional_params["stream_options"]
## check if custom pricing set ##
if any(litellm_params.get(key) is not None for key in _CUSTOM_PRICING_KEYS & litellm_params.keys()):
if any(
litellm_params.get(key) is not None
for key in _CUSTOM_PRICING_KEYS & litellm_params.keys()
):
self.custom_pricing = True
if "custom_llm_provider" in self.model_call_details:
@ -1633,11 +1638,19 @@ class Logging(LiteLLMLoggingBaseClass):
"standard_logging_object"
)
) is not None:
standard_logging_payload["response"] = (
response_dict = (
result.model_dump()
if hasattr(result, "model_dump")
else dict(result)
)
# Ensure usage is properly included with transformed chat format
if transformed_usage is not None:
response_dict["usage"] = (
transformed_usage.model_dump()
if hasattr(transformed_usage, "model_dump")
else dict(transformed_usage)
)
standard_logging_payload["response"] = response_dict
elif isinstance(result, TranscriptionResponse):
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
TranscriptionUsageObjectTransformation,
@ -2323,18 +2336,28 @@ class Logging(LiteLLMLoggingBaseClass):
batch_cost = kwargs.get("batch_cost", None)
batch_usage = kwargs.get("batch_usage", None)
batch_models = kwargs.get("batch_models", None)
if all([batch_cost, batch_usage, batch_models]) is not None:
has_explicit_batch_data = all(
x is not None for x in (batch_cost, batch_usage, batch_models)
)
should_compute_batch_data = (
not is_base64_unified_file_id
or not has_explicit_batch_data
and result.status == "completed"
)
if has_explicit_batch_data:
result._hidden_params["response_cost"] = batch_cost
result._hidden_params["batch_models"] = batch_models
result.usage = batch_usage
elif not is_base64_unified_file_id: # only run for non-unified file ids
elif should_compute_batch_data:
(
response_cost,
batch_usage,
batch_models,
) = await _handle_completed_batch(
batch=result, custom_llm_provider=self.custom_llm_provider
batch=result,
custom_llm_provider=self.custom_llm_provider,
)
result._hidden_params["response_cost"] = response_cost
@ -3299,6 +3322,7 @@ def _get_masked_values(
"token",
"key",
"secret",
"vertex_credentials",
]
return {
k: (
@ -4453,6 +4477,7 @@ class StandardLoggingPayloadSetup:
user_api_key_request_route=None,
spend_logs_metadata=None,
requester_ip_address=None,
user_agent=None,
requester_metadata=None,
prompt_management_metadata=prompt_management_metadata,
applied_guardrails=applied_guardrails,
@ -4533,6 +4558,10 @@ class StandardLoggingPayloadSetup:
)
elif isinstance(usage, Usage):
return usage
elif isinstance(usage, ResponseAPIUsage):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
usage
)
elif isinstance(usage, dict):
if ResponseAPILoggingUtils._is_response_api_usage(usage):
return (
@ -4733,7 +4762,14 @@ class StandardLoggingPayloadSetup:
) -> StandardLoggingPayloadErrorInformation:
from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG
error_status: str = str(getattr(original_exception, "status_code", ""))
# Check for 'code' first (used by ProxyException), then fall back to 'status_code' (used by LiteLLM exceptions)
# Ensure error_code is always a string for Prisma Python JSON field compatibility
error_code_attr = getattr(original_exception, "code", None)
if error_code_attr is not None and str(error_code_attr) not in ("", "None"):
error_status: str = str(error_code_attr)
else:
status_code_attr = getattr(original_exception, "status_code", None)
error_status = str(status_code_attr) if status_code_attr is not None else ""
error_class: str = (
str(original_exception.__class__.__name__) if original_exception else ""
)
@ -5138,6 +5174,7 @@ def get_standard_logging_object_payload(
model_group=_model_group,
model_id=_model_id,
requester_ip_address=clean_metadata.get("requester_ip_address", None),
user_agent=clean_metadata.get("user_agent", None),
messages=StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=kwargs, messages=kwargs.get("messages")
),
@ -5203,6 +5240,7 @@ def get_standard_logging_metadata(
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
user_agent=None,
requester_metadata=None,
user_api_key_end_user_id=None,
prompt_management_metadata=None,

40
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -23,6 +23,15 @@ def _is_above_128k(tokens: float) -> bool:
return False
def get_billable_input_tokens(usage: Usage) -> int:
"""
Returns the number of billable input tokens.
Subtracts cached tokens from prompt tokens if applicable.
"""
details = _parse_prompt_tokens_details(usage)
return usage.prompt_tokens - details["cache_hit_tokens"]
def select_cost_metric_for_model(
model_info: ModelInfo,
) -> Literal["cost_per_character", "cost_per_token"]:
@ -190,7 +199,6 @@ def _get_token_base_cost(
1000 if "k" in threshold_str else 1
)
if usage.prompt_tokens > threshold:
prompt_base_cost = cast(
float, _get_cost_per_unit(model_info, key, prompt_base_cost)
)
@ -566,14 +574,28 @@ def generic_cost_per_token( # noqa: PLR0915
if usage.prompt_tokens_details:
prompt_tokens_details = _parse_prompt_tokens_details(usage)
## EDGE CASE - text tokens not set inside PromptTokensDetails
## EDGE CASE - text tokens not set or includes cached tokens (double-counting)
## Some providers (like xAI) report text_tokens = prompt_tokens (including cached)
## We detect this when: text_tokens + cached_tokens + other > prompt_tokens
## Ref: https://github.com/BerriAI/litellm/issues/19680, #14874, #14875
if prompt_tokens_details["text_tokens"] == 0:
cache_hit = prompt_tokens_details["cache_hit_tokens"]
text_tokens = prompt_tokens_details["text_tokens"]
audio_tokens = prompt_tokens_details["audio_tokens"]
cache_creation = prompt_tokens_details["cache_creation_tokens"]
image_tokens = prompt_tokens_details["image_tokens"]
# Check for double-counting: sum of details > prompt_tokens means overlap
total_details = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens
has_double_counting = cache_hit > 0 and total_details > usage.prompt_tokens
if text_tokens == 0 or has_double_counting:
text_tokens = (
usage.prompt_tokens
- prompt_tokens_details["cache_hit_tokens"]
- prompt_tokens_details["audio_tokens"]
- prompt_tokens_details["cache_creation_tokens"]
- cache_hit
- audio_tokens
- cache_creation
- image_tokens
)
prompt_tokens_details["text_tokens"] = text_tokens
@ -619,7 +641,11 @@ def generic_cost_per_token( # noqa: PLR0915
# Calculate text tokens as remainder when we have a breakdown
# This handles cases like OpenAI's reasoning models where text_tokens isn't provided
text_tokens = max(
0, usage.completion_tokens - reasoning_tokens - audio_tokens - image_tokens
0,
usage.completion_tokens
- reasoning_tokens
- audio_tokens
- image_tokens,
)
else:
# No breakdown at all, all tokens are text tokens

18
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -21,11 +21,13 @@ from litellm.types.utils import (
ChatCompletionMessageToolCall,
ChatCompletionRedactedThinkingBlock,
Choices,
CompletionTokensDetailsWrapper,
Delta,
EmbeddingResponse,
Function,
HiddenParams,
ImageResponse,
PromptTokensDetailsWrapper,
)
from litellm.types.utils import Logprobs as TextCompletionLogprobs
from litellm.types.utils import (
@ -304,6 +306,22 @@ class LiteLLMResponseObjectHandler:
"text_tokens": 0,
}
# Map Responses API naming to Chat Completions API naming for cost calculator
if usage.get("prompt_tokens") is None:
usage["prompt_tokens"] = usage.get("input_tokens", 0)
if usage.get("completion_tokens") is None:
usage["completion_tokens"] = usage.get("output_tokens", 0)
# Convert dicts to wrapper objects so getattr() works in cost calculation
if isinstance(usage.get("input_tokens_details"), dict):
usage["prompt_tokens_details"] = PromptTokensDetailsWrapper(
**usage["input_tokens_details"]
)
if isinstance(usage.get("output_tokens_details"), dict):
usage["completion_tokens_details"] = CompletionTokensDetailsWrapper(
**usage["output_tokens_details"]
)
if model_response_object is None:
model_response_object = ImageResponse(**response_object)
return model_response_object

22
litellm/litellm_core_utils/logging_worker.py
View file

@ -415,6 +415,28 @@ class LoggingWorker:
"""
Safely log a message during shutdown, suppressing errors if logging is closed.
"""
# Check if logger has valid handlers before attempting to log
# During shutdown, handlers may be closed, causing ValueError when writing
if not hasattr(verbose_logger, 'handlers') or not verbose_logger.handlers:
return
# Check if any handler has a valid stream
has_valid_handler = False
for handler in verbose_logger.handlers:
try:
if hasattr(handler, 'stream') and handler.stream and not handler.stream.closed:
has_valid_handler = True
break
elif not hasattr(handler, 'stream'):
# Non-stream handlers (like NullHandler) are always valid
has_valid_handler = True
break
except (AttributeError, ValueError):
continue
if not has_valid_handler:
return
try:
if level == "debug":
verbose_logger.debug(message)

33
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1632,6 +1632,7 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
def convert_to_anthropic_tool_result(
message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
force_base64: bool = False,
) -> AnthropicMessagesToolResultParam:
"""
OpenAI message with a tool result looks like:
@ -1677,13 +1678,16 @@ def convert_to_anthropic_tool_result(
] = []
for content in content_list:
if content["type"] == "text":
anthropic_content_list.append(
AnthropicMessagesToolResultContent(
type="text",
text=content["text"],
cache_control=content.get("cache_control", None),
)
)
# Only include cache_control if explicitly set and not None
# to avoid sending "cache_control": null which breaks some API channels
text_content: AnthropicMessagesToolResultContent = {
"type": "text",
"text": content["text"],
}
cache_control_value = content.get("cache_control")
if cache_control_value is not None:
text_content["cache_control"] = cache_control_value
anthropic_content_list.append(text_content)
elif content["type"] == "image_url":
format = (
content["image_url"].get("format")
@ -1691,7 +1695,7 @@ def convert_to_anthropic_tool_result(
else None
)
_anthropic_image_param = create_anthropic_image_param(
content["image_url"], format=format
content["image_url"], format=format, is_bedrock_invoke=force_base64
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
@ -2053,6 +2057,12 @@ def anthropic_messages_pt( # noqa: PLR0915
else:
messages.append(DEFAULT_USER_CONTINUE_MESSAGE_TYPED)
# Bedrock invoke models have format: invoke/...
# Vertex AI Anthropic also doesn't support URL sources for images
is_bedrock_invoke = model.lower().startswith("invoke/")
is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
force_base64 = is_bedrock_invoke or is_vertex_ai
msg_i = 0
while msg_i < len(messages):
user_content: List[AnthropicMessagesUserMessageValues] = []
@ -2162,7 +2172,9 @@ def anthropic_messages_pt( # noqa: PLR0915
):
# OpenAI's tool message content will always be a string
user_content.append(
convert_to_anthropic_tool_result(user_message_types_block)
convert_to_anthropic_tool_result(
user_message_types_block, force_base64=force_base64
)
)
msg_i += 1
@ -4408,7 +4420,7 @@ def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]:
]
"""
"""
Bedrock toolConfig looks like:
Bedrock toolConfig looks like:
"tools": [
{
"toolSpec": {
@ -4436,6 +4448,7 @@ def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]:
tool_block_list: List[BedrockToolBlock] = []
for tool in tools:
# Handle regular function tools
parameters = tool.get("function", {}).get(
"parameters", {"type": "object", "properties": {}}
)

14
litellm/litellm_core_utils/prompt_templates/image_handling.py
View file

@ -31,15 +31,19 @@ def _process_image_response(response: Response, url: str) -> str:
f"Error: Image size ({size_mb:.2f}MB) exceeds maximum allowed size ({MAX_IMAGE_URL_DOWNLOAD_SIZE_MB}MB). url={url}"
)
image_bytes = response.content
# Stream download with size checking to prevent downloading huge files
max_bytes = int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * 1024 * 1024)
image_bytes = bytearray()
bytes_downloaded = 0
# Check actual size after download if Content-Length was not available
if content_length is None:
size_mb = len(image_bytes) / (1024 * 1024)
if size_mb > MAX_IMAGE_URL_DOWNLOAD_SIZE_MB:
for chunk in response.iter_bytes(chunk_size=8192):
bytes_downloaded += len(chunk)
if bytes_downloaded > max_bytes:
size_mb = bytes_downloaded / (1024 * 1024)
raise litellm.ImageFetchError(
f"Error: Image size ({size_mb:.2f}MB) exceeds maximum allowed size ({MAX_IMAGE_URL_DOWNLOAD_SIZE_MB}MB). url={url}"
)
image_bytes.extend(chunk)
base64_image = base64.b64encode(image_bytes).decode("utf-8")

14
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -110,6 +110,10 @@ class AnthropicMessagesHandler(BaseTranslation):
inputs["tools"] = tools_to_check
if structured_messages:
inputs["structured_messages"] = structured_messages
# Include model information if available
model = data.get("model")
if model:
inputs["model"] = model
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
@ -309,6 +313,14 @@ class AnthropicMessagesHandler(BaseTranslation):
inputs["images"] = images_to_check
if tool_calls_to_check:
inputs["tool_calls"] = tool_calls_to_check
# Include model information from the response if available
response_model = None
if isinstance(response, dict):
response_model = response.get("model")
elif hasattr(response, "model"):
response_model = getattr(response, "model", None)
if response_model:
inputs["model"] = response_model
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
@ -552,7 +564,7 @@ class AnthropicMessagesHandler(BaseTranslation):
response_content = response.get("content", [])
else:
response_content = getattr(response, "content", None) or []
if not response_content:
return False
for content_block in response_content:

19
litellm/llms/anthropic/chat/transformation.py
View file

@ -290,10 +290,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
elif tool_choice == "none":
_tool_choice = AnthropicMessagesToolChoice(type="none")
elif isinstance(tool_choice, dict):
_tool_name = tool_choice.get("function", {}).get("name")
_tool_choice = AnthropicMessagesToolChoice(type="tool")
if _tool_name is not None:
_tool_choice["name"] = _tool_name
if "type" in tool_choice and "function" not in tool_choice:
tool_type = tool_choice.get("type")
if tool_type == "auto":
_tool_choice = AnthropicMessagesToolChoice(type="auto")
elif tool_type == "required" or tool_type == "any":
_tool_choice = AnthropicMessagesToolChoice(type="any")
elif tool_type == "none":
_tool_choice = AnthropicMessagesToolChoice(type="none")
else:
_tool_name = tool_choice.get("function", {}).get("name")
if _tool_name is not None:
_tool_choice = AnthropicMessagesToolChoice(type="tool")
_tool_choice["name"] = _tool_name
if parallel_tool_use is not None:
# Anthropic uses 'disable_parallel_tool_use' flag to determine if parallel tool use is allowed
@ -1369,7 +1378,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
else 0
)
completion_token_details = CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else None,
reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else 0,
text_tokens=completion_tokens - reasoning_tokens if reasoning_tokens > 0 else completion_tokens,
)
total_tokens = prompt_tokens + completion_tokens

179
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -168,6 +168,36 @@ class LiteLLMAnthropicMessagesAdapter:
return provider_specific_fields.get("signature")
return None
def _add_cache_control_if_applicable(
self,
source: Any,
target: Any,
model: Optional[str],
) -> None:
"""
Extract cache_control from source and add to target if it should be preserved.
This method accepts Any type to support both regular dicts and TypedDict objects.
TypedDict objects (like ChatCompletionTextObject, ChatCompletionImageObject, etc.)
are dicts at runtime but have specific types at type-check time. Using Any allows
this method to work with both while maintaining runtime correctness.
Args:
source: Dict or TypedDict containing potential cache_control field
target: Dict or TypedDict to add cache_control to
model: Model name to check if cache_control should be preserved
"""
# TypedDict objects are dicts at runtime, so .get() works
cache_control = source.get("cache_control") if isinstance(source, dict) else getattr(source, "cache_control", None)
if cache_control and model and self.is_anthropic_claude_model(model):
# TypedDict objects support dict operations at runtime
# Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432)
if isinstance(target, dict):
target["cache_control"] = cache_control # type: ignore[typeddict-item]
else:
# Fallback for non-dict objects (shouldn't happen in practice)
cast(Dict[str, Any], target)["cache_control"] = cache_control
def translatable_anthropic_params(self) -> List:
"""
Which anthropic params, we need to translate to the openai format.
@ -205,12 +235,8 @@ class LiteLLMAnthropicMessagesAdapter:
text_obj = ChatCompletionTextObject(
type="text", text=content.get("text", "")
)
# Preserve cache_control if present (for prompt caching)
# Only for Anthropic models that support prompt caching
cache_control = content.get("cache_control")
if cache_control and model and self.is_anthropic_claude_model(model):
text_obj["cache_control"] = cache_control # type: ignore
new_user_content_list.append(text_obj)
self._add_cache_control_if_applicable(content, text_obj, model)
new_user_content_list.append(text_obj) # type: ignore
elif content.get("type") == "image":
# Convert Anthropic image format to OpenAI format
source = content.get("source", {})
@ -225,7 +251,24 @@ class LiteLLMAnthropicMessagesAdapter:
image_obj = ChatCompletionImageObject(
type="image_url", image_url=image_url_obj
)
new_user_content_list.append(image_obj)
self._add_cache_control_if_applicable(content, image_obj, model)
new_user_content_list.append(image_obj) # type: ignore
elif content.get("type") == "document":
# Convert Anthropic document format (PDF, etc.) to OpenAI format
source = content.get("source", {})
openai_image_url = (
self._translate_anthropic_image_to_openai(cast(dict, source))
)
if openai_image_url:
image_url_obj = ChatCompletionImageUrlObject(
url=openai_image_url
)
doc_obj = ChatCompletionImageObject(
type="image_url", image_url=image_url_obj
)
self._add_cache_control_if_applicable(content, doc_obj, model)
new_user_content_list.append(doc_obj) # type: ignore
elif content.get("type") == "tool_result":
if "content" not in content:
tool_result = ChatCompletionToolMessage(
@ -233,14 +276,16 @@ class LiteLLMAnthropicMessagesAdapter:
tool_call_id=content.get("tool_use_id", ""),
content="",
)
tool_message_list.append(tool_result)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result) # type: ignore[arg-type]
elif isinstance(content.get("content"), str):
tool_result = ChatCompletionToolMessage(
role="tool",
tool_call_id=content.get("tool_use_id", ""),
content=str(content.get("content", "")),
)
tool_message_list.append(tool_result)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result) # type: ignore[arg-type]
elif isinstance(content.get("content"), list):
# Combine all content items into a single tool message
# to avoid creating multiple tool_result blocks with the same ID
@ -256,7 +301,8 @@ class LiteLLMAnthropicMessagesAdapter:
tool_call_id=content.get("tool_use_id", ""),
content=c,
)
tool_message_list.append(tool_result)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result) # type: ignore[arg-type]
elif isinstance(c, dict):
if c.get("type") == "text":
tool_result = ChatCompletionToolMessage(
@ -266,7 +312,8 @@ class LiteLLMAnthropicMessagesAdapter:
),
content=c.get("text", ""),
)
tool_message_list.append(tool_result)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result) # type: ignore[arg-type]
elif c.get("type") == "image":
source = c.get("source", {})
openai_image_url = (
@ -282,7 +329,8 @@ class LiteLLMAnthropicMessagesAdapter:
),
content=openai_image_url,
)
tool_message_list.append(tool_result)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result) # type: ignore[arg-type]
else:
# For multiple content items, combine into a single tool message
# with list content to preserve all items while having one tool_use_id
@ -331,7 +379,8 @@ class LiteLLMAnthropicMessagesAdapter:
tool_call_id=content.get("tool_use_id", ""),
content=combined_content_parts, # type: ignore
)
tool_message_list.append(tool_result)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result) # type: ignore[arg-type]
if len(tool_message_list) > 0:
new_messages.extend(tool_message_list)
@ -344,6 +393,8 @@ class LiteLLMAnthropicMessagesAdapter:
## ASSISTANT MESSAGE ##
assistant_message_str: Optional[str] = None
assistant_content_list: List[Dict[str, Any]] = [] # For content blocks with cache_control
has_cache_control_in_text = False
tool_calls: List[ChatCompletionAssistantToolCall] = []
thinking_blocks: List[
Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
@ -357,10 +408,14 @@ class LiteLLMAnthropicMessagesAdapter:
assistant_message_str = str(content)
elif isinstance(content, dict):
if content.get("type") == "text":
if assistant_message_str is None:
assistant_message_str = content.get("text", "")
else:
assistant_message_str += content.get("text", "")
text_block: Dict[str, Any] = {
"type": "text",
"text": content.get("text", ""),
}
self._add_cache_control_if_applicable(content, text_block, model)
if "cache_control" in text_block:
has_cache_control_in_text = True
assistant_content_list.append(text_block)
elif content.get("type") == "tool_use":
function_chunk: ChatCompletionToolCallFunctionChunk = {
"name": content.get("name", ""),
@ -384,13 +439,13 @@ class LiteLLMAnthropicMessagesAdapter:
provider_specific_fields
)
tool_calls.append(
ChatCompletionAssistantToolCall(
id=content.get("id", ""),
type="function",
function=function_chunk,
)
tool_call = ChatCompletionAssistantToolCall(
id=content.get("id", ""),
type="function",
function=function_chunk,
)
self._add_cache_control_if_applicable(content, tool_call, model)
tool_calls.append(tool_call)
elif content.get("type") == "thinking":
thinking_block = ChatCompletionThinkingBlock(
type="thinking",
@ -411,18 +466,30 @@ class LiteLLMAnthropicMessagesAdapter:
if (
assistant_message_str is not None
or len(assistant_content_list) > 0
or len(tool_calls) > 0
or len(thinking_blocks) > 0
):
# Use list format if any text block has cache_control, otherwise use string
if has_cache_control_in_text and len(assistant_content_list) > 0:
assistant_content: Any = assistant_content_list
elif len(assistant_content_list) > 0 and not has_cache_control_in_text:
# Concatenate text blocks into string when no cache_control
assistant_content = "".join(
block.get("text", "") for block in assistant_content_list
)
else:
assistant_content = assistant_message_str
assistant_message = ChatCompletionAssistantMessage(
role="assistant",
content=assistant_message_str,
content=assistant_content,
thinking_blocks=(
thinking_blocks if len(thinking_blocks) > 0 else None
),
)
if len(tool_calls) > 0:
assistant_message["tool_calls"] = tool_calls
assistant_message["tool_calls"] = tool_calls # type: ignore
if len(thinking_blocks) > 0:
assistant_message["thinking_blocks"] = thinking_blocks # type: ignore
new_messages.append(assistant_message)
@ -532,10 +599,10 @@ class LiteLLMAnthropicMessagesAdapter:
)
def translate_anthropic_tools_to_openai(
self, tools: List[AllAnthropicToolsValues]
self, tools: List[AllAnthropicToolsValues], model: Optional[str] = None
) -> List[ChatCompletionToolParam]:
new_tools: List[ChatCompletionToolParam] = []
mapped_tool_params = ["name", "input_schema", "description"]
mapped_tool_params = ["name", "input_schema", "description", "cache_control"]
for tool in tools:
function_chunk = ChatCompletionToolParamFunctionChunk(
name=tool["name"],
@ -548,11 +615,11 @@ class LiteLLMAnthropicMessagesAdapter:
for k, v in tool.items():
if k not in mapped_tool_params: # pass additional computer kwargs
function_chunk.setdefault("parameters", {}).update({k: v})
new_tools.append(
ChatCompletionToolParam(type="function", function=function_chunk)
)
tool_param = ChatCompletionToolParam(type="function", function=function_chunk)
self._add_cache_control_if_applicable(tool, tool_param, model)
new_tools.append(tool_param) # type: ignore[arg-type]
return new_tools
return new_tools # type: ignore[return-value]
def translate_anthropic_output_format_to_openai(
self, output_format: Any
@ -590,6 +657,41 @@ class LiteLLMAnthropicMessagesAdapter:
},
}
def _add_system_message_to_messages(
self,
new_messages: List[AllMessageValues],
anthropic_message_request: AnthropicMessagesRequest,
) -> None:
"""Add system message to messages list if present in request."""
if "system" not in anthropic_message_request:
return
system_content = anthropic_message_request["system"]
if not system_content:
return
# Handle system as string or array of content blocks
if isinstance(system_content, str):
new_messages.insert(
0,
ChatCompletionSystemMessage(role="system", content=system_content),
)
elif isinstance(system_content, list):
# Convert Anthropic system content blocks to OpenAI format
openai_system_content: List[Dict[str, Any]] = []
model_name = anthropic_message_request.get("model", "")
for block in system_content:
if isinstance(block, dict) and block.get("type") == "text":
text_block: Dict[str, Any] = {
"type": "text",
"text": block.get("text", ""),
}
self._add_cache_control_if_applicable(block, text_block, model_name)
openai_system_content.append(text_block)
if openai_system_content:
new_messages.insert(
0,
ChatCompletionSystemMessage(role="system", content=openai_system_content), # type: ignore
)
def translate_anthropic_to_openai(
self, anthropic_message_request: AnthropicMessagesRequest
) -> ChatCompletionRequest:
@ -618,13 +720,7 @@ class LiteLLMAnthropicMessagesAdapter:
model=anthropic_message_request.get("model"),
)
## ADD SYSTEM MESSAGE TO MESSAGES
if "system" in anthropic_message_request:
system_content = anthropic_message_request["system"]
if system_content:
new_messages.insert(
0,
ChatCompletionSystemMessage(role="system", content=system_content),
)
self._add_system_message_to_messages(new_messages, anthropic_message_request)
new_kwargs: ChatCompletionRequest = {
"model": anthropic_message_request["model"],
@ -655,7 +751,8 @@ class LiteLLMAnthropicMessagesAdapter:
tools = anthropic_message_request["tools"]
if tools:
new_kwargs["tools"] = self.translate_anthropic_tools_to_openai(
tools=cast(List[AllAnthropicToolsValues], tools)
tools=cast(List[AllAnthropicToolsValues], tools),
model=new_kwargs.get("model"),
)
## CONVERT THINKING
@ -843,7 +940,7 @@ class LiteLLMAnthropicMessagesAdapter:
role="assistant",
model=response.model or "unknown-model",
stop_sequence=None,
usage=anthropic_usage,
usage=anthropic_usage, # type: ignore
content=anthropic_content, # type: ignore
stop_reason=anthropic_finish_reason,
)
@ -992,7 +1089,7 @@ class LiteLLMAnthropicMessagesAdapter:
else:
usage_delta = UsageDelta(input_tokens=0, output_tokens=0)
return MessageBlockDelta(
type="message_delta", delta=delta, usage=usage_delta
type="message_delta", delta=delta, usage=usage_delta # type: ignore
)
(
type_of_content,

23
litellm/llms/azure/batches/handler.py
View file

@ -5,12 +5,10 @@ Azure Batches API Handler
from typing import Any, Coroutine, Optional, Union, cast
import httpx
from openai import AsyncOpenAI, OpenAI
from litellm.llms.azure.azure import AsyncAzureOpenAI, AzureOpenAI
from litellm.types.llms.openai import (
Batch,
CancelBatchRequest,
CreateBatchRequest,
RetrieveBatchRequest,
@ -130,9 +128,9 @@ class AzureBatchesAPI(BaseAzureLLM):
self,
cancel_batch_data: CancelBatchRequest,
client: Union[AsyncAzureOpenAI, AsyncOpenAI],
) -> Batch:
) -> LiteLLMBatch:
response = await client.batches.cancel(**cancel_batch_data)
return response
return LiteLLMBatch(**response.model_dump())
def cancel_batch(
self,
@ -160,8 +158,23 @@ class AzureBatchesAPI(BaseAzureLLM):
raise ValueError(
"OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
)
if _is_async is True:
if not isinstance(azure_client, (AsyncAzureOpenAI, AsyncOpenAI)):
raise ValueError(
"Azure client is not an instance of AsyncAzureOpenAI or AsyncOpenAI. Make sure you passed an async client."
)
return self.acancel_batch( # type: ignore
cancel_batch_data=cancel_batch_data, client=azure_client
)
# At this point, azure_client is guaranteed to be a sync client
if not isinstance(azure_client, (AzureOpenAI, OpenAI)):
raise ValueError(
"Azure client is not an instance of AzureOpenAI or OpenAI. Make sure you passed a sync client."
)
response = azure_client.batches.cancel(**cancel_batch_data)
return response
return LiteLLMBatch(**response.model_dump())
async def alist_batches(
self,

8
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -22,7 +22,8 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
Accepts both explicit gpt-5 model names and the ``gpt5_series/`` prefix
used for manual routing.
"""
return "gpt-5" in model or "gpt5_series" in model
# gpt-5-chat* is a chat model and shouldn't go through GPT-5 reasoning restrictions.
return ("gpt-5" in model and "gpt-5-chat" not in model) or "gpt5_series" in model
def get_supported_openai_params(self, model: str) -> List[str]:
"""Get supported parameters for Azure OpenAI GPT-5 models.
@ -37,6 +38,11 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
"""
params = OpenAIGPT5Config.get_supported_openai_params(self, model=model)
# Azure supports tool_choice for GPT-5 deployments, but the base GPT-5 config
# can drop it when the deployment name isn't in the OpenAI model registry.
if "tool_choice" not in params:
params.append("tool_choice")
# Only gpt-5.2 has been verified to support logprobs on Azure
if self.is_model_gpt_5_2_model(model):
azure_supported_params = ["logprobs", "top_logprobs"]

37
litellm/llms/azure/cost_calculation.py
View file

@ -1,11 +1,12 @@
"""
Helper util for handling azure openai-specific cost calculation
- e.g.: prompt caching
- e.g.: prompt caching, audio tokens
"""
from typing import Optional, Tuple
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import Usage
from litellm.utils import get_model_info
@ -18,34 +19,15 @@ def cost_per_token(
Input:
- model: str, the model name without provider prefix
- usage: LiteLLM Usage block, containing anthropic caching information
- usage: LiteLLM Usage block, containing caching and audio token information
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
"""
## GET MODEL INFO
model_info = get_model_info(model=model, custom_llm_provider="azure")
cached_tokens: Optional[int] = None
## CALCULATE INPUT COST
non_cached_text_tokens = usage.prompt_tokens
if usage.prompt_tokens_details and usage.prompt_tokens_details.cached_tokens:
cached_tokens = usage.prompt_tokens_details.cached_tokens
non_cached_text_tokens = non_cached_text_tokens - cached_tokens
prompt_cost: float = non_cached_text_tokens * model_info["input_cost_per_token"]
## CALCULATE OUTPUT COST
completion_cost: float = (
usage["completion_tokens"] * model_info["output_cost_per_token"]
)
## Prompt Caching cost calculation
if model_info.get("cache_read_input_token_cost") is not None and cached_tokens:
# Note: We read ._cache_read_input_tokens from the Usage - since cost_calculator.py standardizes the cache read tokens on usage._cache_read_input_tokens
prompt_cost += cached_tokens * (
model_info.get("cache_read_input_token_cost", 0) or 0
)
## Speech / Audio cost calculation
## Speech / Audio cost calculation (cost per second for TTS models)
if (
"output_cost_per_second" in model_info
and model_info["output_cost_per_second"] is not None
@ -55,7 +37,14 @@ def cost_per_token(
f"For model={model} - output_cost_per_second: {model_info.get('output_cost_per_second')}; response time: {response_time_ms}"
)
## COST PER SECOND ##
prompt_cost = 0
prompt_cost = 0.0
completion_cost = model_info["output_cost_per_second"] * response_time_ms / 1000
return prompt_cost, completion_cost
return prompt_cost, completion_cost
## Use generic cost calculator for all other cases
## This properly handles: text tokens, audio tokens, cached tokens, reasoning tokens, etc.
return generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider="azure",
)

26
litellm/llms/base_llm/vector_store/transformation.py
View file

@ -5,8 +5,8 @@ import httpx
from litellm.types.router import GenericLiteLLMParams
from litellm.types.vector_stores import (
BaseVectorStoreAuthCredentials,
VECTOR_STORE_OPENAI_PARAMS,
BaseVectorStoreAuthCredentials,
VectorStoreCreateOptionalRequestParams,
VectorStoreCreateResponse,
VectorStoreIndexEndpoints,
@ -64,6 +64,30 @@ class BaseVectorStoreConfig:
pass
async def atransform_search_vector_store_request(
self,
vector_store_id: str,
query: Union[str, List[str]],
vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
api_base: str,
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> Tuple[str, Dict]:
"""
Optional async version of transform_search_vector_store_request.
If not implemented, the handler will fall back to the sync version.
Providers that need to make async calls (e.g., generating embeddings) should override this.
"""
# Default implementation: call the sync version
return self.transform_search_vector_store_request(
vector_store_id=vector_store_id,
query=query,
vector_store_search_optional_params=vector_store_search_optional_params,
api_base=api_base,
litellm_logging_obj=litellm_logging_obj,
litellm_params=litellm_params,
)
@abstractmethod
def transform_search_vector_store_response(
self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj

2
litellm/llms/bedrock/base_aws_llm.py
View file

@ -1163,7 +1163,7 @@ class BaseAWSLLM:
def _sign_request(
self,
service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore"],
service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore", "s3vectors"],
headers: dict,
optional_params: dict,
request_data: dict,

120
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -76,6 +76,13 @@ BEDROCK_COMPUTER_USE_TOOLS = [
"text_editor_",
]
# Beta header patterns that are not supported by Bedrock Converse API
# These will be filtered out to prevent errors
UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS = [
"advanced-tool-use", # Bedrock Converse doesn't support advanced-tool-use beta headers
"prompt-caching", # Prompt caching not supported in Converse API
]
class AmazonConverseConfig(BaseConfig):
"""
@ -298,6 +305,39 @@ class AmazonConverseConfig(BaseConfig):
# Check if the model is specifically Nova Lite 2
return "nova-2-lite" in model_without_region
def _map_web_search_options(
self,
web_search_options: dict,
model: str
) -> Optional[BedrockToolBlock]:
"""
Map web_search_options to Nova grounding systemTool.
Nova grounding (web search) is only supported on Amazon Nova models.
Returns None for non-Nova models.
Args:
web_search_options: The web_search_options dict from the request
model: The model identifier string
Returns:
BedrockToolBlock with systemTool for Nova models, None otherwise
Reference: https://docs.aws.amazon.com/nova/latest/userguide/grounding.html
"""
# Only Nova models support nova_grounding
# Model strings can be like: "amazon.nova-pro-v1:0", "us.amazon.nova-pro-v1:0", etc.
if "nova" not in model.lower():
verbose_logger.debug(
f"web_search_options passed but model {model} is not a Nova model. "
"Nova grounding is only supported on Amazon Nova models."
)
return None
# Nova doesn't support search_context_size or user_location params
# (unlike Anthropic), so we just enable grounding with no options
return BedrockToolBlock(systemTool={"name": "nova_grounding"})
def _transform_reasoning_effort_to_reasoning_config(
self, reasoning_effort: str
) -> dict:
@ -438,6 +478,10 @@ class AmazonConverseConfig(BaseConfig):
):
supported_params.append("tools")
# Nova models support web_search_options (mapped to nova_grounding systemTool)
if base_model.startswith("amazon.nova"):
supported_params.append("web_search_options")
if litellm.utils.supports_tool_choice(
model=model, custom_llm_provider=self.custom_llm_provider
) or litellm.utils.supports_tool_choice(
@ -573,6 +617,37 @@ class AmazonConverseConfig(BaseConfig):
return transformed_tools
def _filter_unsupported_beta_headers_for_bedrock(
self, model: str, beta_list: list
) -> list:
"""
Remove beta headers that are not supported on Bedrock Converse API for the given model.
Extended thinking beta headers are only supported on specific Claude 4+ models.
Some beta headers are universally unsupported on Bedrock Converse API.
Args:
model: The model name
beta_list: The list of beta headers to filter
Returns:
Filtered list of beta headers
"""
filtered_betas = []
# 1. Filter out beta headers that are universally unsupported on Bedrock Converse
for beta in beta_list:
should_keep = True
for unsupported_pattern in UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS:
if unsupported_pattern in beta.lower():
should_keep = False
break
if should_keep:
filtered_betas.append(beta)
return filtered_betas
def _separate_computer_use_tools(
self, tools: List[OpenAIChatCompletionToolParam], model: str
) -> Tuple[
@ -730,6 +805,13 @@ class AmazonConverseConfig(BaseConfig):
if bedrock_tier in ("default", "flex", "priority"):
optional_params["serviceTier"] = {"type": bedrock_tier}
if param == "web_search_options" and value and isinstance(value, dict):
grounding_tool = self._map_web_search_options(value, model)
if grounding_tool is not None:
optional_params = self._add_tools_to_optional_params(
optional_params=optional_params, tools=[grounding_tool]
)
# Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models
# Nova Lite 2 handles token budgeting differently through reasoningConfig
if "gpt-oss" not in model and not self._is_nova_lite_2_model(model):
@ -1044,7 +1126,14 @@ class AmazonConverseConfig(BaseConfig):
if beta not in seen:
unique_betas.append(beta)
seen.add(beta)
additional_request_params["anthropic_beta"] = unique_betas
# Filter out unsupported beta headers for Bedrock Converse API
filtered_betas = self._filter_unsupported_beta_headers_for_bedrock(
model=model,
beta_list=unique_betas,
)
additional_request_params["anthropic_beta"] = filtered_betas
return bedrock_tools, anthropic_beta_list
@ -1388,20 +1477,23 @@ class AmazonConverseConfig(BaseConfig):
str,
List[ChatCompletionToolCallChunk],
Optional[List[BedrockConverseReasoningContentBlock]],
Optional[List[CitationsContentBlock]],
]:
"""
Translate the message content to a string and a list of tool calls and reasoning content blocks
Translate the message content to a string and a list of tool calls, reasoning content blocks, and citations.
Returns:
content_str: str
tools: List[ChatCompletionToolCallChunk]
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]]
citationsContentBlocks: Optional[List[CitationsContentBlock]] - Citations from Nova grounding
"""
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
for idx, content in enumerate(content_blocks):
"""
- Content is either a tool response or text
@ -1446,10 +1538,15 @@ class AmazonConverseConfig(BaseConfig):
if reasoningContentBlocks is None:
reasoningContentBlocks = []
reasoningContentBlocks.append(content["reasoningContent"])
# Handle Nova grounding citations content
if "citationsContent" in content:
if citationsContentBlocks is None:
citationsContentBlocks = []
citationsContentBlocks.append(content["citationsContent"])
return content_str, tools, reasoningContentBlocks
return content_str, tools, reasoningContentBlocks, citationsContentBlocks
def _transform_response(
def _transform_response( # noqa: PLR0915
self,
model: str,
response: httpx.Response,
@ -1525,18 +1622,27 @@ class AmazonConverseConfig(BaseConfig):
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
if message is not None:
(
content_str,
tools,
reasoningContentBlocks,
citationsContentBlocks,
) = self._translate_message_content(message["content"])
# Initialize provider_specific_fields if we have any special content blocks
provider_specific_fields: dict = {}
if reasoningContentBlocks is not None:
provider_specific_fields["reasoningContentBlocks"] = reasoningContentBlocks
if citationsContentBlocks is not None:
provider_specific_fields["citationsContent"] = citationsContentBlocks
if provider_specific_fields:
chat_completion_message["provider_specific_fields"] = provider_specific_fields
if reasoningContentBlocks is not None:
chat_completion_message["provider_specific_fields"] = {
"reasoningContentBlocks": reasoningContentBlocks,
}
chat_completion_message["reasoning_content"] = (
self._transform_reasoning_content(reasoningContentBlocks)
)

5
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -1476,6 +1476,11 @@ class AWSEventStreamDecoder:
reasoning_content = (
"" # set to non-empty string to ensure consistency with Anthropic
)
elif "citationsContent" in delta_obj:
# Handle Nova grounding citations in streaming responses
provider_specific_fields = {
"citationsContent": delta_obj["citationsContent"],
}
return (
text,
tool_use,

37
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -53,13 +53,26 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
model: str,
drop_params: bool,
) -> dict:
return AnthropicConfig.map_openai_params(
# Force tool-based structured outputs for Bedrock Invoke
# (similar to VertexAI fix in #19201)
# Bedrock Invoke doesn't support output_format parameter
original_model = model
if "response_format" in non_default_params:
# Use a model name that forces tool-based approach
model = "claude-3-sonnet-20240229"
optional_params = AnthropicConfig.map_openai_params(
self,
non_default_params,
optional_params,
model,
drop_params,
)
# Restore original model name
model = original_model
return optional_params
def transform_request(
@ -90,6 +103,8 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
_anthropic_request.pop("model", None)
_anthropic_request.pop("stream", None)
# Bedrock Invoke doesn't support output_format parameter
_anthropic_request.pop("output_format", None)
if "anthropic_version" not in _anthropic_request:
_anthropic_request["anthropic_version"] = self.anthropic_version
@ -117,6 +132,26 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
if "opus-4" in model.lower() or "opus_4" in model.lower():
beta_set.add("tool-search-tool-2025-10-19")
# Filter out beta headers that Bedrock Invoke doesn't support
# AWS Bedrock only supports a specific whitelist of beta flags
# Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html
BEDROCK_SUPPORTED_BETAS = {
"computer-use-2024-10-22", # Legacy computer use
"computer-use-2025-01-24", # Current computer use (Claude 3.7 Sonnet)
"token-efficient-tools-2025-02-19", # Tool use (Claude 3.7+ and Claude 4+)
"interleaved-thinking-2025-05-14", # Interleaved thinking (Claude 4+)
"output-128k-2025-02-19", # 128K output tokens (Claude 3.7 Sonnet)
"dev-full-thinking-2025-05-14", # Developer mode for raw thinking (Claude 4+)
"context-1m-2025-08-07", # 1 million tokens (Claude Sonnet 4)
"context-management-2025-06-27", # Context management (Claude Sonnet/Haiku 4.5)
"effort-2025-11-24", # Effort parameter (Claude Opus 4.5)
"tool-search-tool-2025-10-19", # Tool search (Claude Opus 4.5)
"tool-examples-2025-10-29", # Tool use examples (Claude Opus 4.5)
}
# Only keep beta headers that Bedrock supports
beta_set = {beta for beta in beta_set if beta in BEDROCK_SUPPORTED_BETAS}
if beta_set:
_anthropic_request["anthropic_beta"] = list(beta_set)

23
litellm/llms/bedrock/common_utils.py
View file

@ -797,7 +797,7 @@ class BedrockEventStreamDecoderBase:
def get_anthropic_beta_from_headers(headers: dict) -> List[str]:
"""
Extract anthropic-beta header values and convert them to a list.
Supports comma-separated values from user headers.
Supports both JSON array format and comma-separated values from user headers.
Used by both converse and invoke transformations for consistent handling
of anthropic-beta headers that should be passed to AWS Bedrock.
@ -812,8 +812,25 @@ def get_anthropic_beta_from_headers(headers: dict) -> List[str]:
if not anthropic_beta_header:
return []
# Split comma-separated values and strip whitespace
return [beta.strip() for beta in anthropic_beta_header.split(",")]
# If it's already a list, return it
if isinstance(anthropic_beta_header, list):
return anthropic_beta_header
# Try to parse as JSON array first (e.g., '["interleaved-thinking-2025-05-14", "claude-code-20250219"]')
if isinstance(anthropic_beta_header, str):
anthropic_beta_header = anthropic_beta_header.strip()
if anthropic_beta_header.startswith("[") and anthropic_beta_header.endswith("]"):
try:
parsed = json.loads(anthropic_beta_header)
if isinstance(parsed, list):
return [str(beta).strip() for beta in parsed]
except json.JSONDecodeError:
pass # Fall through to comma-separated parsing
# Fall back to comma-separated values
return [beta.strip() for beta in anthropic_beta_header.split(",")]
return []
class CommonBatchFilesUtils:

68
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -54,6 +54,7 @@ class AmazonAnthropicClaudeMessagesConfig(
# These will be filtered out to prevent 400 "invalid beta flag" errors
UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS = [
"advanced-tool-use", # Bedrock Invoke doesn't support advanced-tool-use beta headers
"prompt-caching-scope"
]
def __init__(self, **kwargs):
@ -162,6 +163,49 @@ class AmazonAnthropicClaudeMessagesConfig(
return any(pattern in model_lower for pattern in supported_patterns)
def _is_claude_opus_4_5(self, model: str) -> bool:
"""
Check if the model is Claude Opus 4.5.
Args:
model: The model name
Returns:
True if the model is Claude Opus 4.5
"""
model_lower = model.lower()
opus_4_5_patterns = [
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5",
]
return any(pattern in model_lower for pattern in opus_4_5_patterns)
def _supports_tool_search_on_bedrock(self, model: str) -> bool:
"""
Check if the model supports tool search on Bedrock.
On Amazon Bedrock, server-side tool search is supported on Claude Opus 4.5
and Claude Sonnet 4.5 with the tool-search-tool-2025-10-19 beta header.
Ref: https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-search-tool
Args:
model: The model name
Returns:
True if the model supports tool search on Bedrock
"""
model_lower = model.lower()
# Supported models for tool search on Bedrock
supported_patterns = [
# Opus 4.5
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5",
# Sonnet 4.5
"sonnet-4.5", "sonnet_4.5", "sonnet-4-5", "sonnet_4_5",
]
return any(pattern in model_lower for pattern in supported_patterns)
def _filter_unsupported_beta_headers_for_bedrock(
self, model: str, beta_set: set
) -> None:
@ -169,25 +213,33 @@ class AmazonAnthropicClaudeMessagesConfig(
Remove beta headers that are not supported on Bedrock for the given model.
Extended thinking beta headers are only supported on specific Claude 4+ models.
Advanced tool use headers are not supported on Bedrock Invoke API.
Advanced tool use headers are not supported on Bedrock Invoke API, but need to be
translated to Bedrock-specific headers for models that support tool search
(Claude Opus 4.5, Sonnet 4.5).
This prevents 400 "invalid beta flag" errors on Bedrock.
Note: Bedrock Invoke API fails with a 400 error when unsupported beta headers
are sent, returning: {"message":"invalid beta flag"}
Translation for models supporting tool search (Opus 4.5, Sonnet 4.5):
- advanced-tool-use-2025-11-20 -> tool-search-tool-2025-10-19 + tool-examples-2025-10-29
Args:
model: The model name
beta_set: The set of beta headers to filter in-place
"""
beta_headers_to_remove = set()
has_advanced_tool_use = False
# 1. Filter out beta headers that are universally unsupported on Bedrock Invoke
# 1. Filter out beta headers that are universally unsupported on Bedrock Invoke and track if advanced-tool-use header is present
for beta in beta_set:
for unsupported_pattern in self.UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS:
if unsupported_pattern in beta.lower():
beta_headers_to_remove.add(beta)
has_advanced_tool_use = True
break
# 2. Filter out extended thinking headers for models that don't support them
extended_thinking_patterns = [
"extended-thinking",
@ -204,6 +256,14 @@ class AmazonAnthropicClaudeMessagesConfig(
for beta in beta_headers_to_remove:
beta_set.discard(beta)
# 3. Translate advanced-tool-use to Bedrock-specific headers for models that support tool search
# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html
# Ref: https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-search-tool
if has_advanced_tool_use and self._supports_tool_search_on_bedrock(model):
beta_set.add("tool-search-tool-2025-10-19")
beta_set.add("tool-examples-2025-10-29")
def _get_tool_search_beta_header_for_bedrock(
self,
model: str,
@ -256,7 +316,7 @@ class AmazonAnthropicClaudeMessagesConfig(
Ref: https://aws.amazon.com/blogs/machine-learning/structured-data-response-with-amazon-bedrock-prompt-engineering-and-tool-use/
"""
import json
# Extract schema from output_format
schema = output_format.get("schema")
if not schema:

8
litellm/llms/cohere/rerank/guardrail_translation/handler.py
View file

@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Optional
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
@ -49,8 +50,13 @@ class CohereRerankHandler(BaseTranslation):
# Process query only
query = data.get("query")
if query is not None and isinstance(query, str):
inputs = GenericGuardrailAPIInputs(texts=[query])
# Include model information if available
model = data.get("model")
if model:
inputs["model"] = model
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs={"texts": [query]},
inputs=inputs,
request_data=data,
input_type="request",
logging_obj=litellm_logging_obj,

36
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -7033,17 +7033,31 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
(
url,
request_body,
) = vector_store_provider_config.transform_search_vector_store_request(
vector_store_id=vector_store_id,
query=query,
vector_store_search_optional_params=vector_store_search_optional_params,
api_base=api_base,
litellm_logging_obj=logging_obj,
litellm_params=dict(litellm_params),
)
# Check if provider has async transform method
if hasattr(vector_store_provider_config, "atransform_search_vector_store_request"):
(
url,
request_body,
) = await vector_store_provider_config.atransform_search_vector_store_request(
vector_store_id=vector_store_id,
query=query,
vector_store_search_optional_params=vector_store_search_optional_params,
api_base=api_base,
litellm_logging_obj=logging_obj,
litellm_params=dict(litellm_params),
)
else:
(
url,
request_body,
) = vector_store_provider_config.transform_search_vector_store_request(
vector_store_id=vector_store_id,
query=query,
vector_store_search_optional_params=vector_store_search_optional_params,
api_base=api_base,
litellm_logging_obj=logging_obj,
litellm_params=dict(litellm_params),
)
all_optional_params: Dict[str, Any] = dict(litellm_params)
all_optional_params.update(vector_store_search_optional_params or {})
headers, signed_json_body = vector_store_provider_config.sign_request(

2
litellm/llms/gemini/chat/transformation.py
View file

@ -92,7 +92,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
"parallel_tool_calls",
"web_search_options",
]
if supports_reasoning(model):
if supports_reasoning(model, custom_llm_provider="gemini"):
supported_params.append("reasoning_effort")
supported_params.append("thinking")
if self.is_model_gemini_audio_model(model):

146
litellm/llms/gemini/files/transformation.py
View file

@ -35,6 +35,26 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.GEMINI
def validate_environment(
self,
api_key: Optional[str],
headers: dict,
model: str,
messages: list,
optional_params: dict,
litellm_params: dict,
) -> dict:
"""
Validate environment and add Gemini API key to headers.
Google AI Studio uses x-goog-api-key header for authentication.
"""
api_key = self.get_api_key(api_key)
if not api_key:
raise ValueError("GEMINI_API_KEY is required for Google AI Studio file operations")
headers["x-goog-api-key"] = api_key
return headers
def get_complete_url(
self,
api_base: Optional[str],
@ -56,10 +76,12 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
if not api_base:
raise ValueError("api_base is required")
if not api_key:
# Get API key from multiple sources
final_api_key = api_key or litellm_params.get("api_key") or self.get_api_key()
if not final_api_key:
raise ValueError("api_key is required")
url = "{}/{}?key={}".format(api_base, endpoint, api_key)
url = "{}/{}?key={}".format(api_base, endpoint, final_api_key)
return url
def get_supported_openai_params(
@ -180,7 +202,25 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file retrieval")
"""
Get the URL to retrieve a file from Google AI Studio.
We expect file_id to be the URI (e.g. https://generativelanguage.googleapis.com/v1beta/files/...)
as returned by the upload response.
"""
api_key = litellm_params.get("api_key")
if not api_key:
raise ValueError("api_key is required")
if file_id.startswith("http"):
url = "{}?key={}".format(file_id, api_key)
else:
# Fallback for just file name (files/...)
api_base = self.get_api_base(litellm_params.get("api_base")) or "https://generativelanguage.googleapis.com"
api_base = api_base.rstrip("/")
url = "{}/v1beta/{}?key={}".format(api_base, file_id, api_key)
return url, {"Content-Type": "application/json"}
def transform_retrieve_file_response(
self,
@ -188,7 +228,40 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file retrieval")
"""
Transform Gemini's file retrieval response into OpenAI-style FileObject
"""
try:
response_json = raw_response.json()
# Map Gemini state to OpenAI status
gemini_state = response_json.get("state", "STATE_UNSPECIFIED")
status = "uploaded" # Default
if gemini_state == "ACTIVE":
status = "processed"
elif gemini_state == "FAILED":
status = "error"
return OpenAIFileObject(
id=response_json.get("uri", ""),
bytes=int(response_json.get("sizeBytes", 0)),
created_at=int(
time.mktime(
time.strptime(
response_json["createTime"].replace("Z", "+00:00"),
"%Y-%m-%dT%H:%M:%S.%f%z",
)
)
),
filename=response_json.get("displayName", ""),
object="file",
purpose="user_data",
status=status,
status_details=str(response_json.get("error", "")) if gemini_state == "FAILED" else None,
)
except Exception as e:
verbose_logger.exception(f"Error parsing file retrieve response: {str(e)}")
raise ValueError(f"Error parsing file retrieve response: {str(e)}")
def transform_delete_file_request(
self,
@ -196,7 +269,41 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file deletion")
"""
Transform delete file request for Google AI Studio.
Args:
file_id: The file URI (e.g., "files/abc123" or full URI)
optional_params: Optional parameters
litellm_params: LiteLLM parameters containing api_key
Returns:
tuple[str, dict]: (url, params) for the DELETE request
"""
api_base = self.get_api_base(litellm_params.get("api_base"))
if not api_base:
raise ValueError("api_base is required")
# Get API key from multiple sources (same pattern as get_complete_url)
api_key = litellm_params.get("api_key") or self.get_api_key()
if not api_key:
raise ValueError("api_key is required")
# Extract file name from URI if full URI is provided
# file_id could be "files/abc123" or "https://generativelanguage.googleapis.com/v1beta/files/abc123"
if file_id.startswith("http"):
# Extract the file path from full URI
file_name = file_id.split("/v1beta/")[-1]
else:
file_name = file_id
# Construct the delete URL
url = f"{api_base}/v1beta/{file_name}"
# Add API key as header (Google AI Studio uses x-goog-api-key header)
params = {}
return url, params
def transform_delete_file_response(
self,
@ -204,7 +311,34 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> FileDeleted:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file deletion")
"""
Transform Gemini's file delete response into OpenAI-style FileDeleted.
Google AI Studio returns an empty JSON object {} on successful deletion.
"""
try:
# Google AI Studio returns {} on successful deletion
if raw_response.status_code == 200:
# Extract file ID from the request URL if possible
file_id = "deleted"
if hasattr(raw_response, "request") and raw_response.request:
url = str(raw_response.request.url)
if "/files/" in url:
file_id = url.split("/files/")[-1].split("?")[0]
# Add the files/ prefix if not present
if not file_id.startswith("files/"):
file_id = f"files/{file_id}"
return FileDeleted(
id=file_id,
deleted=True,
object="file"
)
else:
raise ValueError(f"Failed to delete file: {raw_response.text}")
except Exception as e:
verbose_logger.exception(f"Error parsing file delete response: {str(e)}")
raise ValueError(f"Error parsing file delete response: {str(e)}")
def transform_list_files_request(
self,

5
litellm/llms/gemini/image_edit/transformation.py
View file

@ -106,7 +106,10 @@ class GeminiImageEditConfig(BaseImageEditConfig):
generation_config: Dict[str, Any] = {}
if "aspectRatio" in image_edit_optional_request_params:
generation_config["aspectRatio"] = image_edit_optional_request_params[
# Move aspectRatio into imageConfig inside generationConfig
if "imageConfig" not in generation_config:
generation_config["imageConfig"] = {}
generation_config["imageConfig"]["aspectRatio"] = image_edit_optional_request_params[
"aspectRatio"
]

56
litellm/llms/gigachat/chat/transformation.py
View file

@ -31,6 +31,16 @@ else:
GIGACHAT_BASE_URL = "https://gigachat.devices.sberbank.ru/api/v1"
def is_valid_json(value: str) -> bool:
"""Checks whether the value passed is a valid serialized JSON string"""
try:
json.loads(value)
except json.JSONDecodeError:
return False
else:
return True
class GigaChatError(BaseLLMException):
"""GigaChat API error."""
@ -101,7 +111,11 @@ class GigaChatConfig(BaseConfig):
Set up headers with OAuth token.
"""
# Get access token
credentials = api_key or get_secret_str("GIGACHAT_CREDENTIALS") or get_secret_str("GIGACHAT_API_KEY")
credentials = (
api_key
or get_secret_str("GIGACHAT_CREDENTIALS")
or get_secret_str("GIGACHAT_API_KEY")
)
access_token = get_access_token(credentials=credentials)
# Store credentials for image uploads
@ -193,11 +207,13 @@ class GigaChatConfig(BaseConfig):
for tool in tools:
if tool.get("type") == "function":
func = tool.get("function", {})
functions.append({
"name": func.get("name", ""),
"description": func.get("description", ""),
"parameters": func.get("parameters", {}),
})
functions.append(
{
"name": func.get("name", ""),
"description": func.get("description", ""),
"parameters": func.get("parameters", {}),
}
)
return functions
def _map_tool_choice(
@ -281,8 +297,14 @@ class GigaChatConfig(BaseConfig):
}
# Add optional params
for key in ["temperature", "top_p", "max_tokens", "stream",
"repetition_penalty", "profanity_check"]:
for key in [
"temperature",
"top_p",
"max_tokens",
"stream",
"repetition_penalty",
"profanity_check",
]:
if key in optional_params:
request_data[key] = optional_params[key]
@ -314,7 +336,7 @@ class GigaChatConfig(BaseConfig):
elif role == "tool":
message["role"] = "function"
content = message.get("content", "")
if not isinstance(content, str):
if not isinstance(content, str) or not is_valid_json(content):
message["content"] = json.dumps(content, ensure_ascii=False)
# Handle None content
@ -441,14 +463,16 @@ class GigaChatConfig(BaseConfig):
# Convert to tool_calls format
if isinstance(args, dict):
args = json.dumps(args, ensure_ascii=False)
message_data["tool_calls"] = [{
"id": f"call_{uuid.uuid4().hex[:24]}",
"type": "function",
"function": {
"name": func_call.get("name", ""),
"arguments": args,
message_data["tool_calls"] = [
{
"id": f"call_{uuid.uuid4().hex[:24]}",
"type": "function",
"function": {
"name": func_call.get("name", ""),
"arguments": args,
},
}
}]
]
message_data.pop("function_call", None)
finish_reason = "tool_calls"

8
litellm/llms/groq/chat/transformation.py
View file

@ -323,4 +323,12 @@ class GroqChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler):
status_code=error.get("code"), message=error.get("message"), body=error
)
# Map Groq's 'reasoning' field to LiteLLM's 'reasoning_content' field
# Groq returns delta.reasoning, but LiteLLM expects delta.reasoning_content
choices = chunk.get("choices", [])
for choice in choices:
delta = choice.get("delta", {})
if "reasoning" in delta:
delta["reasoning_content"] = delta.pop("reasoning")
return super().chunk_parser(chunk)

Some files were not shown because too many files have changed in this diff Show more