scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge branch 'main' into litellm_fix_langfuse_otel_trace_v2

Browse source
This commit is contained in:
Harshit Jain 2026-02-20 23:03:18 +05:30 • committed by GitHub
commit e29c339211
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
986 changed files with 59494 additions and 14052 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.circleci/config.yml
View file

@ -21,9 +21,7 @@ commands:
- run:
name: "Install local version of litellm-enterprise"
command: |
cd enterprise
python -m pip install -e .
cd ..
pip install --force-reinstall --no-deps -e enterprise/
setup_litellm_test_deps:
steps:
- checkout

36
.claude/settings.json Normal file
View file

@ -0,0 +1,36 @@
{
"permissions": {
"allow": [
"Bash(git show:*)",
"Bash(git worktree add:*)",
"Read(//Users/krrishdholakia/Documents/litellm/**)",
"Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types/**)",
"Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/**)",
"Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/**)",
"Bash(python:*)",
"Bash(python -c \"\nimport sys; sys.path.insert\\(0, ''.''\\)\nfrom litellm.proxy.guardrails.guardrail_hooks.claude_code.guardrail import ClaudeCodeGuardrail, HOSTED_TOOL_PREFIXES\nprint\\(''HOSTED_TOOL_PREFIXES:'', HOSTED_TOOL_PREFIXES\\)\nprint\\(''ClaudeCodeGuardrail imported OK''\\)\n\")",
"Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy/**)",
"Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/**)",
"Bash(poetry run pytest:*)",
"Bash(git add:*)",
"Bash(git commit:*)",
"Bash(poetry run python:*)",
"Bash(poetry run pip:*)",
"Bash(git reset:*)",
"Bash(git cherry-pick:*)",
"Bash(git checkout:*)",
"Read(//Users/krrishdholakia/Documents/litellm/litellm/proxy/guardrails/guardrail_hooks/**)",
"Read(//Users/krrishdholakia/Documents/**)",
"Bash(git -C /Users/krrishdholakia/Documents/litellm-mcp-user-permissions worktree list)",
"Bash(ls:*)"
],
"additionalDirectories": [
"/Users/krrishdholakia/Documents/litellm-mcp-group-plan/plan",
"/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/proxy/guardrails/guardrail_hooks/claude_code",
"/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types",
"/Users/krrishdholakia/Documents/litellm-claude-code-guardrails",
"/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy",
"/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/tests/test_litellm/proxy/auth"
]
}
}

80
.github/workflows/regenerate-poetry-lock.yml vendored Normal file
View file

@ -0,0 +1,80 @@
name: Regenerate poetry.lock
# Runs whenever pyproject.toml is merged into main (the most common cause of
# the "pyproject.toml changed significantly since poetry.lock was last generated"
# CI failure). Can also be triggered manually.
on:
push:
branches:
- main
paths:
- pyproject.toml
workflow_dispatch:
permissions:
contents: write # needed to push the auto/regenerate-poetry-lock-* branch
pull-requests: write # needed to open the PR and enable auto-merge
jobs:
regenerate-lock:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install Poetry
run: pip install poetry
- name: Regenerate poetry.lock
run: poetry lock
- name: Check whether poetry.lock actually changed
id: diff
run: |
if git diff --quiet poetry.lock; then
echo "changed=false" >> "$GITHUB_OUTPUT"
else
echo "changed=true" >> "$GITHUB_OUTPUT"
fi
- name: Open PR with the refreshed lock file
if: steps.diff.outputs.changed == 'true'
id: open-pr
run: |
BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')"
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git checkout -b "$BRANCH"
git add poetry.lock
git commit -m "chore: regenerate poetry.lock to match pyproject.toml"
git push -f origin "$BRANCH"
cat > /tmp/pr-body.md << 'BODY'
Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`.
Fixes the recurring CI failure:
```
pyproject.toml changed significantly since poetry.lock was last generated.
Run `poetry lock` to fix the lock file.
```
BODY
PR_URL=$(gh pr create \
--title "chore: regenerate poetry.lock to match pyproject.toml" \
--body-file /tmp/pr-body.md \
--head "$BRANCH" \
--base main)
echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
env:
GH_TOKEN: ${{ github.token }}
- name: Enable auto-merge
if: steps.diff.outputs.changed == 'true'
run: |
gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash
env:
GH_TOKEN: ${{ github.token }}

47
.github/workflows/test-litellm-matrix.yml vendored
View file

@ -12,44 +12,59 @@ concurrency:
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 15
timeout-minutes: 20 # Increased from 15 to 20
strategy:
fail-fast: false
matrix:
test-group:
# tests/test_litellm split by subdirectory (~560 files total)
- name: "llms"
path: "tests/test_litellm/llms"
workers: 4
# Vertex AI tests separated for better isolation (prevent auth/env pollution)
- name: "llms-vertex"
path: "tests/test_litellm/llms/vertex_ai"
workers: 1
reruns: 2
- name: "llms-other"
path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
workers: 2
reruns: 2
# tests/test_litellm/proxy split by subdirectory (~180 files total)
- name: "proxy-guardrails"
path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
workers: 4
workers: 2
reruns: 2
- name: "proxy-core"
path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
workers: 4
workers: 2
reruns: 2
- name: "proxy-misc"
path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
workers: 4
workers: 2
reruns: 2
- name: "integrations"
path: "tests/test_litellm/integrations"
workers: 4
workers: 2
reruns: 3 # Integration tests tend to be flakier
- name: "core-utils"
path: "tests/test_litellm/litellm_core_utils"
workers: 2
reruns: 1
- name: "other"
path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types"
workers: 4
workers: 2
reruns: 2
- name: "root"
path: "tests/test_litellm/test_*.py"
workers: 4
workers: 2
reruns: 2
# tests/proxy_unit_tests split alphabetically (~48 files total)
- name: "proxy-unit-a"
path: "tests/proxy_unit_tests/test_[a-o]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b"
path: "tests/proxy_unit_tests/test_[p-z]*.py"
workers: 2
reruns: 1
name: test (${{ matrix.test-group.name }})
@ -79,12 +94,17 @@ jobs:
run: |
poetry config virtualenvs.in-project true
poetry install --with dev,proxy-dev --extras "proxy semantic-router"
poetry run pip install pytest-retry==1.6.3 pytest-xdist google-genai==1.22.0 \
# pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
poetry run pip install google-genai==1.22.0 \
google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core
- name: Setup litellm-enterprise
run: |
cd enterprise && poetry run pip install -e . && cd ..
poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Generate Prisma client
run: |
poetry run prisma generate --schema litellm/proxy/schema.prisma
- name: Run tests - ${{ matrix.test-group.name }}
run: |
@ -92,4 +112,7 @@ jobs:
--tb=short -vv \
--maxfail=10 \
-n ${{ matrix.test-group.workers }} \
--reruns ${{ matrix.test-group.reruns }} \
--reruns-delay 1 \
--dist=loadscope \
--durations=20

4
.github/workflows/test-litellm.yml vendored
View file

@ -42,9 +42,7 @@ jobs:
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
poetry run pip install -e .
cd ..
poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Run tests
run: |
poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

4
.github/workflows/test-mcp.yml vendored
View file

@ -40,9 +40,7 @@ jobs:
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
python -m pip install -e .
cd ..
poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Run MCP tests
run: |

96
.github/workflows/test_server_root_path.yml vendored Normal file
View file

@ -0,0 +1,96 @@
name: Test Proxy SERVER_ROOT_PATH Routing
permissions:
contents: read
on:
pull_request:
branches: [main]
jobs:
test-server-root-path:
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
matrix:
root_path: ["/api/v1", "/llmproxy"]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build Docker image
uses: docker/build-push-action@v5
with:
context: .
file: ./docker/Dockerfile.non_root
tags: litellm-test:${{ github.sha }}
load: true
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Start LiteLLM container with SERVER_ROOT_PATH
run: |
docker run -d \
--name litellm-test \
-p 4000:4000 \
-e SERVER_ROOT_PATH="${{ matrix.root_path }}" \
-e LITELLM_MASTER_KEY="sk-1234" \
litellm-test:${{ github.sha }} \
--detailed_debug
- name: Wait for container to be healthy
run: |
echo "Waiting for LiteLLM to start..."
max_attempts=30
attempt=0
while [ $attempt -lt $max_attempts ]; do
if docker logs litellm-test 2>&1 | grep -q "Uvicorn running"; then
echo "LiteLLM started successfully"
break
fi
attempt=$((attempt + 1))
echo "Attempt $attempt/$max_attempts - waiting for server to start..."
sleep 2
done
if [ $attempt -eq $max_attempts ]; then
echo "Server failed to start within timeout"
docker logs litellm-test
exit 1
fi
sleep 5
- name: Show container logs
if: always()
run: docker logs litellm-test
- name: Test UI endpoint with root path
run: |
ROOT_PATH="${{ matrix.root_path }}"
echo "Testing UI at: http://localhost:4000${ROOT_PATH}/ui/"
for i in 1 2 3; do
content=$(curl -sL --max-time 5 -H "Authorization: Bearer sk-1234" "http://localhost:4000${ROOT_PATH}/ui/")
if echo "$content" | grep -q -E "(html|<!DOCTYPE|<head|<body)"; then
echo "UI page contains valid HTML content"
exit 0
fi
echo "Attempt $i/3 - no valid HTML, retrying in 5s..."
sleep 5
done
echo "UI page does not contain expected HTML content"
echo "Response: $content"
docker logs litellm-test
exit 1
- name: Cleanup
if: always()
run: |
docker stop litellm-test || true
docker rm litellm-test || true

293
cookbook/mock_prompt_management_server/README.md Normal file
View file

@ -0,0 +1,293 @@
# Mock Prompt Management Server
A reference implementation of the [LiteLLM Generic Prompt Management API](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api).
This FastAPI server demonstrates how to build a prompt management API that integrates with LiteLLM without requiring a PR to the LiteLLM repository.
## Quick Start
### 1. Install Dependencies
```bash
pip install fastapi uvicorn pydantic
```
### 2. Start the Server
```bash
python mock_prompt_management_server.py
```
The server will start on `http://localhost:8080`
### 3. Test the Endpoint
```bash
# Get a prompt
curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
# Get a prompt with authentication
curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" \
-H "Authorization: Bearer test-token-12345"
# List all prompts
curl "http://localhost:8080/prompts"
# Get prompt variables
curl "http://localhost:8080/prompts/hello-world-prompt/variables"
```
## Using with LiteLLM
### Configuration
Create a `config.yaml` file:
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
prompts:
- prompt_id: "hello-world-prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
api_base: http://localhost:8080
api_key: test-token-12345
```
### Start LiteLLM Proxy
```bash
litellm --config config.yaml
```
### Make a Request
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-3.5-turbo",
"prompt_id": "hello-world-prompt",
"prompt_variables": {
"domain": "data science",
"task": "analyzing customer behavior"
},
"messages": [
{"role": "user", "content": "Please help me get started"}
]
}'
```
## Available Prompts
The server includes several example prompts:
| Prompt ID | Description | Variables |
|-----------|-------------|-----------|
| `hello-world-prompt` | Basic helpful assistant | `domain`, `task` |
| `code-review-prompt` | Code review assistant | `years_experience`, `language`, `code` |
| `customer-support-prompt` | Customer support agent | `company_name`, `customer_message` |
| `data-analysis-prompt` | Data analysis expert | `analysis_type`, `dataset_name`, `data` |
| `creative-writing-prompt` | Creative writing assistant | `genre`, `length`, `topic` |
## Authentication
The server supports optional Bearer token authentication. Valid tokens for testing:
- `test-token-12345`
- `dev-token-67890`
- `prod-token-abcdef`
If no `Authorization` header is provided, requests are allowed (for testing purposes).
## API Endpoints
### LiteLLM Spec Endpoints
#### `GET /beta/litellm_prompt_management`
Get a prompt by ID (required by LiteLLM).
**Query Parameters:**
- `prompt_id` (required): The prompt ID
- `project_name` (optional): Project filter
- `slug` (optional): Slug filter
- `version` (optional): Version filter
**Response:**
```json
{
"prompt_id": "hello-world-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant specialized in {domain}."
},
{
"role": "user",
"content": "Help me with: {task}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 500
}
}
```
### Convenience Endpoints (Not in LiteLLM Spec)
#### `GET /health`
Health check endpoint.
#### `GET /prompts`
List all available prompts.
#### `GET /prompts/{prompt_id}/variables`
Get all variables used in a prompt template.
#### `POST /prompts`
Create a new prompt (in-memory only, for testing).
## Example: Full Integration Test
### 1. Start the Mock Server
```bash
python mock_prompt_management_server.py
```
### 2. Test with Python
```python
from litellm import completion
# The completion will:
# 1. Fetch the prompt from your API
# 2. Replace {domain} with "machine learning"
# 3. Replace {task} with "building a recommendation system"
# 4. Merge with your messages
# 5. Use the model and params from the prompt
response = completion(
model="gpt-4",
prompt_id="hello-world-prompt",
prompt_variables={
"domain": "machine learning",
"task": "building a recommendation system"
},
messages=[
{"role": "user", "content": "I have user behavior data from the past year."}
],
# Configure the generic prompt manager
generic_prompt_config={
"api_base": "http://localhost:8080",
"api_key": "test-token-12345",
}
)
print(response.choices[0].message.content)
```
## Customization
### Adding New Prompts
Edit the `PROMPTS_DB` dictionary in `mock_prompt_management_server.py`:
```python
PROMPTS_DB = {
"my-custom-prompt": {
"prompt_id": "my-custom-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a {role}."
},
{
"role": "user",
"content": "{user_input}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.8,
"max_tokens": 1000
}
}
}
```
### Using a Database
Replace the `PROMPTS_DB` dictionary with database queries:
```python
@app.get("/beta/litellm_prompt_management")
async def get_prompt(prompt_id: str):
# Fetch from database
prompt = await db.prompts.find_one({"prompt_id": prompt_id})
if not prompt:
raise HTTPException(status_code=404, detail="Prompt not found")
return PromptResponse(**prompt)
```
### Adding Access Control
Use the custom query parameters for access control:
```python
@app.get("/beta/litellm_prompt_management")
async def get_prompt(
prompt_id: str,
project_name: Optional[str] = None,
user_id: Optional[str] = None,
authorization: Optional[str] = Header(None)
):
token = verify_api_key(authorization)
# Check if user has access to this project
if not has_project_access(token, project_name):
raise HTTPException(status_code=403, detail="Access denied")
# Fetch and return prompt
...
```
## Production Considerations
Before deploying to production:
1. **Use a real database** instead of in-memory storage
2. **Implement proper authentication** with JWT tokens or API keys
3. **Add rate limiting** to prevent abuse
4. **Use HTTPS** for encrypted communication
5. **Add logging and monitoring** for observability
6. **Implement caching** for frequently accessed prompts
7. **Add versioning** for prompt management
8. **Implement access control** based on teams/users
9. **Add input validation** for all parameters
10. **Use environment variables** for configuration
## Related Documentation
- [Generic Prompt Management API Documentation](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api)
- [LiteLLM Prompt Management](https://docs.litellm.ai/docs/proxy/prompt_management)
- [Generic Guardrail API](https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api)
## Questions?
This is a reference implementation for the LiteLLM Generic Prompt Management API. For questions or issues, please open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm).

390
cookbook/mock_prompt_management_server/mock_prompt_management_server.py Normal file
View file

@ -0,0 +1,390 @@
#!/usr/bin/env python3
"""
Mock Prompt Management API Server
This is a FastAPI server that implements the LiteLLM Generic Prompt Management API
for testing and demonstration purposes.
Usage:
python mock_prompt_management_server.py
The server will start on http://localhost:8080
Test the endpoint:
curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
"""
import os
import json
from typing import Any, Dict, List, Optional
from fastapi import FastAPI, HTTPException, Header, Query, status
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field
# ============================================================================
# Response Models
# ============================================================================
class MessageContent(BaseModel):
"""A single message in the prompt template"""
role: str = Field(..., description="Message role (system, user, assistant)")
content: str = Field(
..., description="Message content with optional {variable} placeholders"
)
class PromptResponse(BaseModel):
"""Response format for the prompt management API"""
prompt_id: str = Field(..., description="The ID of the prompt")
prompt_template: List[MessageContent] = Field(
..., description="Array of messages in OpenAI format"
)
prompt_template_model: Optional[str] = Field(
None, description="Optional model to use for this prompt"
)
prompt_template_optional_params: Optional[Dict[str, Any]] = Field(
None, description="Optional parameters like temperature, max_tokens, etc."
)
# ============================================================================
# Mock Prompt Database
# ============================================================================
PROMPTS_DB = {
"hello-world-prompt": {
"prompt_id": "hello-world-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant specialized in {domain}.",
},
{"role": "user", "content": "Help me with: {task}"},
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {"temperature": 0.7, "max_tokens": 500},
},
"code-review-prompt": {
"prompt_id": "code-review-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are an expert code reviewer with {years_experience} years of experience in {language}.",
},
{
"role": "user",
"content": "Please review the following code for bugs, security issues, and best practices:\n\n{code}",
},
],
"prompt_template_model": "gpt-4-turbo",
"prompt_template_optional_params": {
"temperature": 0.3,
"max_tokens": 1500,
},
},
"customer-support-prompt": {
"prompt_id": "customer-support-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a friendly customer support agent for {company_name}. Always be professional, empathetic, and solution-oriented.",
},
{
"role": "user",
"content": "Customer inquiry: {customer_message}",
},
],
"prompt_template_model": "gpt-3.5-turbo",
"prompt_template_optional_params": {
"temperature": 0.8,
"max_tokens": 800,
"top_p": 0.9,
},
},
"data-analysis-prompt": {
"prompt_id": "data-analysis-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a data scientist expert in {analysis_type} analysis.",
},
{
"role": "user",
"content": "Analyze the following data and provide insights:\n\nDataset: {dataset_name}\nData: {data}",
},
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.5,
"max_tokens": 2000,
},
},
"creative-writing-prompt": {
"prompt_id": "creative-writing-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a creative writer specializing in {genre} fiction.",
},
{
"role": "user",
"content": "Write a {length} story about: {topic}",
},
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.9,
"max_tokens": 3000,
"top_p": 0.95,
},
},
}
# Valid API tokens for authentication (in production, use a secure token store)
VALID_API_TOKENS = {
"test-token-12345",
"dev-token-67890",
"prod-token-abcdef",
}
# ============================================================================
# FastAPI App
# ============================================================================
app = FastAPI(
title="Mock Prompt Management API",
description="A mock server implementing the LiteLLM Generic Prompt Management API",
version="1.0.0",
)
def verify_api_key(authorization: Optional[str] = Header(None)) -> bool:
"""
Verify the API key from the Authorization header.
Args:
authorization: Authorization header (Bearer token)
Returns:
True if valid, raises HTTPException if invalid
"""
if authorization is None:
# Allow requests without authentication for testing
return True
# Extract token from "Bearer <token>"
if not authorization.startswith("Bearer "):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid authorization header format. Expected 'Bearer <token>'",
)
token = authorization.replace("Bearer ", "").strip()
if token not in VALID_API_TOKENS:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid API key",
)
return True
@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
async def get_prompt(
prompt_id: str = Query(..., description="The ID of the prompt to fetch"),
project_name: Optional[str] = Query(
None, description="Optional project name filter"
),
slug: Optional[str] = Query(None, description="Optional slug filter"),
version: Optional[str] = Query(None, description="Optional version filter"),
authorization: Optional[str] = Header(None),
) -> PromptResponse:
"""
Get a prompt by ID with optional filtering.
This endpoint implements the LiteLLM Generic Prompt Management API specification.
Args:
prompt_id: The ID of the prompt to fetch
project_name: Optional project name for filtering
slug: Optional slug for filtering
version: Optional version for filtering
authorization: Optional Bearer token for authentication
Returns:
PromptResponse with the prompt template and configuration
Raises:
HTTPException: 401 if authentication fails, 404 if prompt not found
"""
# Verify authentication
verify_api_key(authorization)
# Log the request parameters (useful for debugging)
print(f"Fetching prompt: {prompt_id}")
if project_name:
print(f" Project: {project_name}")
if slug:
print(f" Slug: {slug}")
if version:
print(f" Version: {version}")
# Check if prompt exists
if prompt_id not in PROMPTS_DB:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Prompt '{prompt_id}' not found. Available prompts: {list(PROMPTS_DB.keys())}",
)
# Get the prompt from the database
prompt_data = PROMPTS_DB[prompt_id]
# Optional: Apply filtering based on project_name, slug, or version
# In a real implementation, you might use these to filter prompts by access control
# or to fetch specific versions from your database
return PromptResponse(**prompt_data)
@app.get("/health")
async def health_check():
"""Health check endpoint"""
return {
"status": "healthy",
"service": "mock-prompt-management-api",
"version": "1.0.0",
}
@app.get("/prompts")
async def list_prompts(authorization: Optional[str] = Header(None)):
"""
List all available prompts.
This is a convenience endpoint (not part of the LiteLLM spec) for
discovering available prompts.
"""
# Verify authentication
verify_api_key(authorization)
prompts_list = [
{
"prompt_id": pid,
"model": p.get("prompt_template_model"),
"has_variables": any(
"{" in msg.get("content", "") for msg in p.get("prompt_template", [])
),
}
for pid, p in PROMPTS_DB.items()
]
return {"prompts": prompts_list, "total": len(prompts_list)}
@app.get("/prompts/{prompt_id}/variables")
async def get_prompt_variables(
prompt_id: str, authorization: Optional[str] = Header(None)
):
"""
Get all variables in a prompt template.
This is a convenience endpoint (not part of the LiteLLM spec) for
discovering what variables a prompt expects.
"""
# Verify authentication
verify_api_key(authorization)
if prompt_id not in PROMPTS_DB:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Prompt '{prompt_id}' not found",
)
prompt_data = PROMPTS_DB[prompt_id]
variables = set()
# Extract variables from the prompt template
import re
for message in prompt_data["prompt_template"]:
content = message.get("content", "")
# Find all {variable} patterns
found_vars = re.findall(r"\{(\w+)\}", content)
variables.update(found_vars)
return {
"prompt_id": prompt_id,
"variables": sorted(list(variables)),
"example_usage": {
"prompt_id": prompt_id,
"prompt_variables": {var: f"<{var}_value>" for var in variables},
},
}
@app.post("/prompts")
async def create_prompt(
prompt: PromptResponse, authorization: Optional[str] = Header(None)
):
"""
Create a new prompt (convenience endpoint for testing).
This is NOT part of the LiteLLM spec - it's just for testing purposes.
"""
# Verify authentication
verify_api_key(authorization)
if prompt.prompt_id in PROMPTS_DB:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Prompt '{prompt.prompt_id}' already exists",
)
PROMPTS_DB[prompt.prompt_id] = prompt.dict()
return {
"status": "created",
"prompt_id": prompt.prompt_id,
"message": "Prompt created successfully (in-memory only)",
}
# ============================================================================
# Main
# ============================================================================
if __name__ == "__main__":
import uvicorn
print("=" * 70)
print("Mock Prompt Management API Server")
print("=" * 70)
print(f"\nStarting server on http://localhost:8080")
print(f"\nAvailable prompts: {len(PROMPTS_DB)}")
for prompt_id in PROMPTS_DB.keys():
print(f" - {prompt_id}")
print(f"\nValid API tokens: {len(VALID_API_TOKENS)}")
print(" - test-token-12345")
print(" - dev-token-67890")
print(" - prod-token-abcdef")
print("\nEndpoints:")
print(" GET /beta/litellm_prompt_management?prompt_id=<id> (LiteLLM spec)")
print(" GET /health (health check)")
print(" GET /prompts (list all prompts)")
print(
" GET /prompts/{id}/variables (get prompt variables)"
)
print(" POST /prompts (create prompt)")
print("\nExample usage:")
print(
' curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"'
)
print("\nPress CTRL+C to stop the server")
print("=" * 70)
uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info")

177
docs/my-website/blog/claude_code_beta_headers/index.md Normal file
View file

@ -0,0 +1,177 @@
---
slug: claude-code-beta-headers-incident
title: "Incident Report: Invalid beta headers with Claude Code"
date: 2026-02-16T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
tags: [incident-report, anthropic, stability]
hide_table_of_contents: false
---
**Date:** February 13, 2026
**Duration:** ~3 hours
**Severity:** High
**Status:** Resolved
> **Note:** This fix will be available starting from `v1.81.13-nightly` or higher of LiteLLM.
## Summary
Claude Code began sending unsupported Anthropic beta headers to non-Anthropic providers (Bedrock, Azure AI, Vertex AI), causing `invalid beta flag` errors. LiteLLM was forwarding all beta headers without provider-specific validation. Users experienced request failures when routing Claude Code requests through LiteLLM to these providers.
- **LLM calls to Anthropic:** No impact.
- **LLM calls to Bedrock/Azure/Vertex:** Failed with `invalid beta flag` errors when unsupported headers were present.
- **Cost tracking and routing:** No impact.
{/* truncate */}
---
## Background
Anthropic uses beta headers to enable experimental features in Claude. When Claude Code makes API requests, it includes headers like `anthropic-beta: prompt-caching-scope-2026-01-05,advanced-tool-use-2025-11-20`. However, not all providers support all Anthropic beta features.
Before this incident, LiteLLM forwarded all beta headers to all providers without validation:
```mermaid
sequenceDiagram
participant CC as Claude Code
participant LP as LiteLLM (old behavior)
participant Provider as Provider (Bedrock/Azure/Vertex)
CC->>LP: Request with beta headers
Note over CC,LP: anthropic-beta: header1,header2,header3
LP->>Provider: Forward ALL headers (no validation)
Note over LP,Provider: anthropic-beta: header1,header2,header3
Provider-->>LP: ❌ Error: invalid beta flag
LP-->>CC: Request fails
```
Requests succeeded for Anthropic (native support) but failed for other providers when Claude Code sent headers those providers didn't support.
---
## Root cause
LiteLLM lacked provider-specific beta header validation. When Claude Code introduced new beta features or sent headers that specific providers didn't support, those headers were blindly forwarded, causing provider API errors.
---
## Remediation
| # | Action | Status | Code |
|---|---|---|---|
| 1 | Create `anthropic_beta_headers_config.json` with provider-specific mappings | ✅ Done | [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) |
| 2 | Implement strict validation: headers must be explicitly mapped to be forwarded | ✅ Done | [`litellm_logging.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/litellm_logging.py) |
| 3 | Add `/reload/anthropic_beta_headers` endpoint for dynamic config updates | ✅ Done | Proxy management endpoints |
| 4 | Add `/schedule/anthropic_beta_headers_reload` for automatic periodic updates | ✅ Done | Proxy management endpoints |
| 5 | Support `LITELLM_ANTHROPIC_BETA_HEADERS_URL` for custom config sources | ✅ Done | Environment configuration |
| 6 | Support `LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS` for air-gapped deployments | ✅ Done | Environment configuration |
Now LiteLLM validates and transforms headers per-provider:
```mermaid
sequenceDiagram
participant CC as Claude Code
participant LP as LiteLLM (new behavior)
participant Config as Beta Headers Config
participant Provider as Provider (Bedrock/Azure/Vertex)
CC->>LP: Request with beta headers
Note over CC,LP: anthropic-beta: header1,header2,header3
LP->>Config: Load header mapping for provider
Config-->>LP: Returns mapping (header→value or null)
Note over LP: Validate & Transform:<br/>1. Check if header exists in mapping<br/>2. Filter out null values<br/>3. Map to provider-specific names
LP->>Provider: Request with filtered & mapped headers
Note over LP,Provider: anthropic-beta: mapped-header2<br/>(header1, header3 filtered out)
Provider-->>LP: ✅ Success response
LP-->>CC: Response
```
---
## Dynamic configuration updates
A key improvement is zero-downtime configuration updates. When Anthropic releases new beta features, users can update their configuration without restarting:
```bash
# Manually trigger reload (no restart needed)
curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
# Or schedule automatic reloads every 24 hours
curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
This prevents future incidents where Claude Code introduces new headers before LiteLLM configuration is updated.
---
## Configuration format
The `anthropic_beta_headers_config.json` file maps input headers to provider-specific output headers:
```json
{
"description": "Mapping of Anthropic beta headers for each provider.",
"anthropic": {
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"computer-use-2025-01-24": "computer-use-2025-01-24"
},
"bedrock_converse": {
"advanced-tool-use-2025-11-20": null,
"computer-use-2025-01-24": "computer-use-2025-01-24"
},
"azure_ai": {
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"computer-use-2025-01-24": "computer-use-2025-01-24"
}
}
```
**Validation rules:**
1. Headers must exist in the mapping for the target provider
2. Headers with `null` values are filtered out (unsupported)
3. Header names can be transformed per-provider (e.g., Bedrock uses different names for some features)
---
## Resolution steps for users
For users still experiencing issues, update to the latest LiteLLM version if < v1.81.11-nightly:
```bash
pip install --upgrade litellm
```
Or manually reload the configuration without restarting:
```bash
curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
---
## Related documentation
- [Managing Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) - Complete configuration guide
- [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) - Current configuration file

2
docs/my-website/blog/claude_opus_4_6/index.md
View file

@ -185,7 +185,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
model_list:
- model_name: claude-opus-4-6
litellm_params:
model: bedrock/anthropic.claude-opus-4-6-v1:0
model: bedrock/anthropic.claude-opus-4-6-v1
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

283
docs/my-website/blog/claude_sonnet_4_6/index.md Normal file
View file

@ -0,0 +1,283 @@
---
slug: claude_sonnet_4_6
title: "Day 0 Support: Claude Sonnet 4.6"
date: 2026-02-17T10:00:00
authors:
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
tags: [anthropic, claude, sonnet 4.6]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
## Docker Image
```bash
docker pull ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6
```
## Usage - Anthropic
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-sonnet-4-6
litellm_params:
model: anthropic/claude-sonnet-4-6
api_key: os.environ/ANTHROPIC_API_KEY
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-sonnet-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
from litellm import completion
response = completion(
model="anthropic/claude-sonnet-4-6",
messages=[{"role": "user", "content": "what llm are you"}]
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
## Usage - Azure
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-sonnet-4-6
litellm_params:
model: azure_ai/claude-sonnet-4-6
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE # https://<resource>.services.ai.azure.com
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
-e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-sonnet-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
from litellm import completion
response = completion(
model="azure_ai/claude-sonnet-4-6",
api_key="your-azure-api-key",
api_base="https://<resource>.services.ai.azure.com",
messages=[{"role": "user", "content": "what llm are you"}]
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
## Usage - Vertex AI
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-sonnet-4-6
litellm_params:
model: vertex_ai/claude-sonnet-4-6
vertex_project: os.environ/VERTEX_PROJECT
vertex_location: us-east5
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e VERTEX_PROJECT=$VERTEX_PROJECT \
-e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
-v $(pwd)/config.yaml:/app/config.yaml \
-v $(pwd)/credentials.json:/app/credentials.json \
ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-sonnet-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
from litellm import completion
response = completion(
model="vertex_ai/claude-sonnet-4-6",
vertex_project="your-project-id",
vertex_location="us-east5",
messages=[{"role": "user", "content": "what llm are you"}]
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
## Usage - Bedrock
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-sonnet-4-6
litellm_params:
model: bedrock/anthropic.claude-sonnet-4-6-v1
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-sonnet-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
from litellm import completion
response = completion(
model="bedrock/anthropic.claude-sonnet-4-6-v1",
aws_access_key_id="your-access-key",
aws_secret_access_key="your-secret-key",
aws_region_name="us-east-1",
messages=[{"role": "user", "content": "what llm are you"}]
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>

150
docs/my-website/blog/gemin_3.1/index.md Normal file
View file

@ -0,0 +1,150 @@
---
slug: gemini_3_1_pro
title: "DAY 0 Support: Gemini 3.1 Pro on LiteLLM"
date: 2026-02-19T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Guide to using Gemini 3.1 Pro on LiteLLM Proxy and SDK with day 0 support."
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Gemini 3.1 Pro Day 0 Support
LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along with it.
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.81.9-stable.gemini.3.1-pro
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==v1.81.9-stable.gemini.3.1-pro
```
</TabItem>
</Tabs>
## What's New
### 1. New Thinking Levels: `thinkingLevel` with MINIMAL & MEDIUM
Gemini 3.1 Pro introduces support for **medium** thinking level
LiteLLM automatically maps the OpenAI `reasoning_effort` parameter to Gemini's `thinkingLevel`, so you can use familiar `reasoning_effort` values (`minimal`, `low`, `medium`, `high`) without changing your code!
---
## Supported Endpoints
LiteLLM provides **full end-to-end support** for Gemini 3.1 Pro on:
- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint
All endpoints support:
- Streaming and non-streaming responses
- Function calling with thought signatures
- Multi-turn conversations
- All Gemini 3-specific features
- Conversion of provider specific thinking related param to thinkingLevel
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
**Basic Usage with MEDIUM thinking (NEW)**
```python
from litellm import completion
# No need to make any changes to your code as we map openai reasoning param to thinkingLevel
response = completion(
model="gemini/gemini-3.1-pro-preview",
messages=[{"role": "user", "content": "Solve this complex math problem: 25 * 4 + 10"}],
reasoning_effort="medium", # NEW: MEDIUM thinking level
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gemini-3.1-pro-preview
litellm_params:
model: gemini/gemini-3.1-pro-preview
api_key: os.environ/GEMINI_API_KEY
- model_name: vertex-gemini-3.1-pro-preview
litellm_params:
model: vertex_ai/gemini-3.1-pro-preview
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
```
**3. Call with MEDIUM thinking**
```bash
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-d '{
"model": "gemini-3.1-pro-preview",
"messages": [{"role": "user", "content": "Complex reasoning task"}],
"reasoning_effort": "medium"
}'
```
</TabItem>
</Tabs>
---
## `reasoning_effort` Mapping for Gemini 3+
| reasoning_effort | thinking_level |
|------------------|----------------|
| `minimal` | `minimal` |
| `low` | `low` |
| `medium` | `medium` |
| `high` | `high` |
| `disable` | `minimal` |
| `none` | `minimal` |

117
docs/my-website/blog/vllm_embeddings_incident/index.md Normal file
View file

@ -0,0 +1,117 @@
---
slug: vllm-embeddings-incident
title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter"
date: 2026-02-18T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
tags: [incident-report, embeddings, vllm]
hide_table_of_contents: false
---
**Date:** Feb 16, 2026
**Duration:** ~3 hours
**Severity:** High (for vLLM embedding users)
**Status:** Resolved
## Summary
A commit ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)) intended to fix OpenAI SDK behavior broke vLLM embeddings by explicitly passing `encoding_format=None` in API requests. vLLM rejects this with error: `"unknown variant \`\`, expected float or base64"`.
- **vLLM embedding calls:** Complete failure - all requests rejected
- **Other providers:** No impact - OpenAI and other providers functioned normally
- **Other vLLM functionality:** No impact - only embeddings were affected
{/* truncate */}
---
## Background
The `encoding_format` parameter for embeddings specifies whether vectors should be returned as `float` arrays or `base64` encoded strings. Different providers have different expectations:
- **OpenAI SDK:** If `encoding_format` is omitted, the SDK adds a default value of `"float"`
- **vLLM:** Strictly validates `encoding_format` - only accepts `"float"`, `"base64"`, or complete omission. Rejects `None` or empty string values.
```mermaid
flowchart TD
A["1. User calls litellm.embedding()
litellm/main.py"] --> B["2. Transform request for provider
litellm/llms/openai_like/embedding/handler.py"]
B --> C["3. Send request to vLLM endpoint"]
C -->|"encoding_format omitted"| D["4a. ✅ vLLM processes request"]
C -->|"encoding_format='float' or 'base64'"| D
C -->|"encoding_format=None or ''"| E["4b. ❌ vLLM rejects with error:
'unknown variant, expected float or base64'"]
style D fill:#d4edda,stroke:#28a745
style E fill:#f8d7da,stroke:#dc3545
style B fill:#fff3cd,stroke:#ffc107
```
---
## Root cause
A well-intentioned fix for OpenAI SDK behavior inadvertently broke vLLM embeddings:
**The Breaking Change ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)):**
In `litellm/main.py`, the code was changed to explicitly set `encoding_format=None` instead of omitting it:
```python
# Added in dbcae4a
if encoding_format is not None:
optional_params["encoding_format"] = encoding_format
else:
# Omitting causes openai sdk to add default value of "float"
optional_params["encoding_format"] = None
```
This fix worked correctly for OpenAI - explicitly passing `None` prevented the SDK from adding its default value. However, vLLM's strict parameter validation rejected `None` values, causing all embedding requests to fail.
---
## The Fix
Fix deployed ([`55348dd`](https://github.com/BerriAI/litellm/commit/55348dd9c51b5b028f676d25ad023b8f052fc071)). The solution filters out `None` and empty string values from `optional_params` before sending requests to OpenAI-like providers (including vLLM).
**In `litellm/llms/openai_like/embedding/handler.py`:**
```python
# Before (broken)
data = {"model": model, "input": input, **optional_params}
# After (fixed)
filtered_optional_params = {k: v for k, v in optional_params.items() if v not in (None, '')}
data = {"model": model, "input": input, **filtered_optional_params}
```
This ensures:
- Valid values (`"float"`, `"base64"`) are preserved and sent
- `None` and empty string values are filtered out (parameter omitted entirely)
- OpenAI SDK no longer adds defaults because liteLLM handles the parameter upstream
---
## Remediation
| # | Action | Status | Code |
|---|---|---|---|
| 1 | Filter `None` and empty string values in OpenAI-like embedding handler | ✅ Done | [`handler.py#L108`](https://github.com/BerriAI/litellm/blob/main/litellm/llms/openai_like/embedding/handler.py#L108) |
| 2 | Unit tests for parameter filtering (None, empty string, valid values) | ✅ Done | [`test_openai_like_embedding.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/openai_like/embedding/test_openai_like_embedding.py) |
| 3 | Transformation tests for hosted_vllm embedding config | ✅ Done | [`test_hosted_vllm_embedding_transformation.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py) |
| 4 | E2E tests with actual vLLM endpoint | ✅ Done | [`test_hosted_vllm_embedding_e2e.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_e2e.py) |
| 5 | Validate JSON payload structure matches vLLM expectations | ✅ Done | Tests verify exact JSON sent to endpoint |
---

1
docs/my-website/docs/adding_provider/generic_guardrail_api.md
View file

@ -237,6 +237,7 @@ litellm_settings:
mode: pre_call # or post_call, during_call
api_base: https://your-guardrail-api.com
api_key: os.environ/YOUR_GUARDRAIL_API_KEY # optional
unreachable_fallback: fail_closed # default: fail_closed. Set to fail_open to proceed if the guardrail endpoint is unreachable (network errors, or HTTP 502/503/504 from an upstream proxy/LB).
additional_provider_specific_params:
# your custom parameters
threshold: 0.8

576
docs/my-website/docs/adding_provider/generic_prompt_management_api.md Normal file
View file

@ -0,0 +1,576 @@
# [BETA] Generic Prompt Management API - Integrate Without a PR
## The Problem
As a prompt management provider, integrating with LiteLLM traditionally requires:
- Making a PR to the LiteLLM repository
- Waiting for review and merge
- Maintaining provider-specific code in LiteLLM's codebase
- Updating the integration for changes to your API
## The Solution
The **Generic Prompt Management API** lets you integrate with LiteLLM **instantly** by implementing a simple API endpoint. No PR required.
### Key Benefits
1. **No PR Needed** - Deploy and integrate immediately
3. **Simple Contract** - One GET endpoint, standard JSON response
4. **Variable Substitution** - Support for prompt variables with `{variable}` syntax
5. **Custom Parameters** - Pass provider-specific query params via config
6. **Full Control** - You own and maintain your prompt management API
7. **Model & Parameters Override** - Optionally override model and parameters from your prompts
## Get Started in 3 Steps
### Step 1: Configure LiteLLM
Add to your `config.yaml`:
```yaml
prompts:
- prompt_id: "simple_prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
api_base: http://localhost:8080
api_key: os.environ/YOUR_API_KEY
```
### Step 2: Implement Your API Endpoint
```python
from fastapi import FastAPI
from pydantic import BaseModel
app = FastAPI()
@app.get("/beta/litellm_prompt_management")
async def get_prompt(prompt_id: str):
return {
"prompt_id": prompt_id,
"prompt_template": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Help me with {task}"}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {"temperature": 0.7}
}
```
### Step 3: Use in Your App
```python
from litellm import completion
response = completion(
model="gpt-4",
prompt_id="simple_prompt",
prompt_variables={"task": "data analysis"},
messages=[{"role": "user", "content": "I have sales data"}]
)
```
That's it! LiteLLM fetches your prompt, applies variables, and makes the request
## API Contract
### Endpoint
Implement `GET /beta/litellm_prompt_management`
### Request Format
Your endpoint will receive a GET request with query parameters:
```
GET /beta/litellm_prompt_management?prompt_id={prompt_id}&{custom_params}
```
**Query Parameters:**
- `prompt_id` (required): The ID of the prompt to fetch
- Custom parameters: Any additional parameters you configured in `provider_specific_query_params`
**Example:**
```
GET /beta/litellm_prompt_management?prompt_id=hello-world-prompt-2bac&project_name=litellm&slug=hello-world-prompt-2bac
```
### Response Format
```json
{
"prompt_id": "hello-world-prompt-2bac",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant specialized in {domain}."
},
{
"role": "user",
"content": "Help me with {task}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 500,
"top_p": 0.9
}
}
```
**Response Fields:**
- `prompt_id` (string, required): The ID of the prompt
- `prompt_template` (array, required): Array of OpenAI-format messages with optional `{variable}` placeholders
- `prompt_template_model` (string, optional): Model to use for this prompt (overrides client model unless `ignore_prompt_manager_model: true`)
- `prompt_template_optional_params` (object, optional): Additional parameters like temperature, max_tokens, etc. (merged with client params unless `ignore_prompt_manager_optional_params: true`)
## LiteLLM Configuration
Add to `config.yaml`:
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
prompts:
- prompt_id: "simple_prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
provider_specific_query_params:
project_name: litellm
slug: hello-world-prompt-2bac
api_base: http://localhost:8080
api_key: os.environ/YOUR_PROMPT_API_KEY # optional
ignore_prompt_manager_model: true # optional, keep client's model
ignore_prompt_manager_optional_params: true # optional, don't merge prompt manager's params (e.g. temperature, max_tokens, etc.)
```
### Configuration Parameters
- `prompt_integration`: Must be `"generic_prompt_management"`
- `provider_specific_query_params`: Custom query parameters sent to your API (optional)
- `api_base`: Base URL of your prompt management API
- `api_key`: Optional API key for authentication (sent as `Bearer` token)
- `ignore_prompt_manager_model`: If `true`, use the model specified by client instead of prompt's model (default: `false`)
- `ignore_prompt_manager_optional_params`: If `true`, don't merge prompt's optional params with client params (default: `false`)
## Usage
### Using with LiteLLM SDK
**Basic usage with prompt ID:**
```python
from litellm import completion
response = completion(
model="gpt-4",
prompt_id="simple_prompt",
messages=[{"role": "user", "content": "Additional message"}]
)
```
**With prompt variables:**
```python
response = completion(
model="gpt-4",
prompt_id="simple_prompt",
prompt_variables={
"domain": "data science",
"task": "analyzing customer churn"
},
messages=[{"role": "user", "content": "Please provide a detailed analysis"}]
)
```
The prompt template will have `{domain}` replaced with "data science" and `{task}` replaced with "analyzing customer churn".
### Using with LiteLLM Proxy
**1. Start the proxy with your config:**
```bash
litellm --config /path/to/config.yaml
```
**2. Make requests with prompt_id:**
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4",
"prompt_id": "simple_prompt",
"prompt_variables": {
"domain": "healthcare",
"task": "patient risk assessment"
},
"messages": [
{"role": "user", "content": "Analyze the following data..."}
]
}'
```
**3. Using with OpenAI SDK:**
```python
from openai import OpenAI
client = OpenAI(
base_url="http://0.0.0.0:4000",
api_key="sk-1234"
)
response = client.chat.completions.create(
model="gpt-4",
messages=[
{"role": "user", "content": "Analyze the data"}
],
extra_body={
"prompt_id": "simple_prompt",
"prompt_variables": {
"domain": "finance",
"task": "fraud detection"
}
}
)
```
## Implementation Example
See [mock_prompt_management_server.py](https://github.com/BerriAI/litellm/blob/main/cookbook/mock_prompt_management_server/mock_prompt_management_server.py) for a complete reference implementation with multiple example prompts, authentication, and convenience endpoints.
**Minimal FastAPI example:**
```python
from fastapi import FastAPI, HTTPException, Header
from typing import Optional, Dict, Any, List
from pydantic import BaseModel
app = FastAPI()
# In-memory prompt storage (replace with your database)
PROMPTS = {
"hello-world-prompt": {
"prompt_id": "hello-world-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant specialized in {domain}."
},
{
"role": "user",
"content": "Help me with: {task}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 500
}
},
"code-review-prompt": {
"prompt_id": "code-review-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are an expert code reviewer. Review code for {language}."
},
{
"role": "user",
"content": "Review the following code:\n\n{code}"
}
],
"prompt_template_model": "gpt-4-turbo",
"prompt_template_optional_params": {
"temperature": 0.3,
"max_tokens": 1000
}
}
}
class PromptResponse(BaseModel):
prompt_id: str
prompt_template: List[Dict[str, str]]
prompt_template_model: Optional[str] = None
prompt_template_optional_params: Optional[Dict[str, Any]] = None
@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
async def get_prompt(
prompt_id: str,
authorization: Optional[str] = Header(None),
project_name: Optional[str] = None,
slug: Optional[str] = None,
):
"""
Get a prompt by ID with optional filtering by project_name and slug.
Args:
prompt_id: The ID of the prompt to fetch
authorization: Optional Bearer token for authentication
project_name: Optional project name filter
slug: Optional slug filter
"""
# Optional: Validate authorization
if authorization:
token = authorization.replace("Bearer ", "")
# Validate your token here
if not is_valid_token(token):
raise HTTPException(status_code=401, detail="Invalid API key")
# Optional: Apply additional filtering based on custom params
if project_name or slug:
# You can use these parameters to filter or validate access
# For example, check if the user has access to this project
pass
# Fetch the prompt from your storage
if prompt_id not in PROMPTS:
raise HTTPException(
status_code=404,
detail=f"Prompt '{prompt_id}' not found"
)
prompt_data = PROMPTS[prompt_id]
return PromptResponse(**prompt_data)
def is_valid_token(token: str) -> bool:
"""Validate API token - implement your logic here"""
# Example: Check against your database or secret store
valid_tokens = ["your-secret-token", "another-valid-token"]
return token in valid_tokens
# Optional: Health check endpoint
@app.get("/health")
async def health_check():
return {"status": "healthy"}
# Optional: List all prompts endpoint
@app.get("/prompts")
async def list_prompts(authorization: Optional[str] = Header(None)):
"""List all available prompts"""
if authorization:
token = authorization.replace("Bearer ", "")
if not is_valid_token(token):
raise HTTPException(status_code=401, detail="Invalid API key")
return {
"prompts": [
{"prompt_id": pid, "model": p.get("prompt_template_model")}
for pid, p in PROMPTS.items()
]
}
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8080)
```
### Running the Example Server
1. Install dependencies:
```bash
pip install fastapi uvicorn
```
2. Save the code above to `prompt_server.py`
3. Run the server:
```bash
python prompt_server.py
```
4. Test the endpoint:
```bash
curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt&project_name=litellm&slug=hello-world-prompt-2bac"
```
Expected response:
```json
{
"prompt_id": "hello-world-prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant specialized in {domain}."
},
{
"role": "user",
"content": "Help me with: {task}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 500
}
}
```
## Advanced Features
### Variable Substitution
LiteLLM automatically substitutes variables in your prompt templates using the `{variable}` syntax. Both `{variable}` and `{{variable}}` formats are supported.
**Example prompt template:**
```json
{
"prompt_template": [
{
"role": "system",
"content": "You are an expert in {domain} with {years} years of experience."
}
]
}
```
**Client request:**
```python
completion(
model="gpt-4",
prompt_id="expert_prompt",
prompt_variables={
"domain": "machine learning",
"years": "10"
}
)
```
**Result:**
```
"You are an expert in machine learning with 10 years of experience."
```
### Caching
LiteLLM automatically caches fetched prompts in memory. The cache key includes:
- `prompt_id`
- `prompt_label` (if provided)
- `prompt_version` (if provided)
This means your API endpoint is only called once per unique prompt configuration.
### Model Override Behavior
**Default behavior (without `ignore_prompt_manager_model`):**
```yaml
prompts:
- prompt_id: "my_prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
api_base: http://localhost:8080
```
If your API returns `"prompt_template_model": "gpt-4"`, LiteLLM will use `gpt-4` regardless of what the client specified.
**With `ignore_prompt_manager_model: true`:**
```yaml
prompts:
- prompt_id: "my_prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
api_base: http://localhost:8080
ignore_prompt_manager_model: true
```
LiteLLM will use the model specified by the client, ignoring the prompt's model.
### Parameter Merging Behavior
**Default behavior (without `ignore_prompt_manager_optional_params`):**
Client params are merged with prompt params, with prompt params taking precedence:
```python
# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
# Client sends: {"temperature": 0.9, "top_p": 0.95}
# Final params: {"temperature": 0.7, "max_tokens": 500, "top_p": 0.95}
```
**With `ignore_prompt_manager_optional_params: true`:**
Only client params are used:
```python
# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
# Client sends: {"temperature": 0.9, "top_p": 0.95}
# Final params: {"temperature": 0.9, "top_p": 0.95}
```
## Security Considerations
1. **Authentication**: Use the `api_key` parameter to secure your prompt management API
2. **Authorization**: Implement team/user-based access control using the custom query parameters
3. **Rate Limiting**: Add rate limiting to prevent abuse of your API
4. **Input Validation**: Validate all query parameters before processing
5. **HTTPS**: Always use HTTPS in production for encrypted communication
6. **Secrets**: Store API keys in environment variables, not in config files
## Use Cases
✅ **Use Generic Prompt Management API when:**
- You want instant integration without waiting for PRs
- You maintain your own prompt management service
- You need full control over prompt versioning and updates
- You want to build custom prompt management features
- You need to integrate with your internal systems
✅ **Common scenarios:**
- Internal prompt management system for your organization
- Multi-tenant prompt management with team-based access control
- A/B testing different prompt versions
- Prompt experimentation and analytics
- Integration with existing prompt engineering workflows
## When to Use This
✅ **Use Generic Prompt Management API when:**
- You want instant integration without waiting for PRs
- You maintain your own prompt management service
- You need full control over updates and features
- You want custom prompt storage and versioning logic
❌ **Make a PR when:**
- You want deeper integration with LiteLLM internals
- Your integration requires complex LiteLLM-specific logic
- You want to be featured as a built-in provider
- You're building a reusable integration for the community
## Troubleshooting
### Prompt not found
- Verify the `prompt_id` matches exactly (case-sensitive)
- Check that your API endpoint is accessible from LiteLLM
- Verify authentication if using `api_key`
### Variables not substituted
- Ensure variables use `{variable}` or `{{variable}}` syntax
- Check that variable names in `prompt_variables` match template exactly
- Variables are case-sensitive
### Model not being overridden
- Check if `ignore_prompt_manager_model: true` is set in config
- Verify your API is returning `prompt_template_model` in the response
### Parameters not being applied
- Check if `ignore_prompt_manager_optional_params: true` is set
- Verify your API is returning `prompt_template_optional_params`
- Ensure parameter names match OpenAI's parameter names
## Questions?
This is a **beta API**. We're actively improving it based on feedback. Open an issue or PR if you need additional capabilities.
## Related Documentation
- [Prompt Management Overview](../proxy/prompt_management.md)
- [Generic Guardrail API](./generic_guardrail_api.md)
- [LiteLLM Proxy Setup](../proxy/quick_start.md)

465
docs/my-website/docs/completion/message_sanitization.md Normal file
View file

@ -0,0 +1,465 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Message Sanitization for Tool Calling for anthropic models
**Automatically fix common message formatting issues when using tool calling with `modify_params=True`**
LiteLLM can automatically sanitize messages to handle common issues that occur during tool calling workflows, especially when using OpenAI-compatible clients with providers that have strict message format requirements (like Anthropic Claude).
## Overview
When `litellm.modify_params = True` is enabled, LiteLLM automatically sanitizes messages to fix three common issues:
1. **Orphaned Tool Calls** - Assistant messages with tool_calls but missing tool results
2. **Orphaned Tool Results** - Tool messages that reference non-existent tool_call_ids
3. **Empty Message Content** - Messages with empty or whitespace-only text content
This ensures your tool calling workflows work seamlessly across different LLM providers without manual message validation.
## Why Message Sanitization?
Different LLM providers have varying requirements for message formats, especially during tool calling:
- **Anthropic Claude** requires every tool_call to have a corresponding tool result
- Some providers reject messages with empty content
- OpenAI-compatible clients may not always maintain perfect message consistency
Without sanitization, these issues cause API errors that interrupt your workflows. With `modify_params=True`, LiteLLM handles these edge cases automatically.
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
# Enable automatic message sanitization
litellm.modify_params = True
# This will work even if messages have formatting issues
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=[
{"role": "user", "content": "What's the weather in Boston?"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_123",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "Boston"}'}
}
]
# Missing tool result - LiteLLM will add a dummy result automatically
},
{"role": "user", "content": "Thanks!"}
],
tools=[{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather for a city",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}]
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
litellm_settings:
modify_params: true # Enable automatic message sanitization
model_list:
- model_name: claude-3-5-sonnet
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
```
</TabItem>
</Tabs>
## Sanitization Cases
### Case A: Orphaned Tool Calls (Missing Tool Results)
**Problem:** An assistant message contains `tool_calls`, but no corresponding tool result messages follow.
**Solution:** LiteLLM automatically adds dummy tool result messages for any missing tool results.
**Example:**
```python
import litellm
litellm.modify_params = True
# Messages with orphaned tool calls
messages = [
{"role": "user", "content": "Search for Python tutorials"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_abc123",
"type": "function",
"function": {"name": "web_search", "arguments": '{"query": "Python tutorials"}'}
}
]
},
# Missing tool result here!
{"role": "user", "content": "What about JavaScript?"}
]
# LiteLLM automatically adds:
# {
# "role": "tool",
# "tool_call_id": "call_abc123",
# "content": "[System: Tool execution skipped/interrupted by user. No result provided for tool 'web_search'.]"
# }
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages,
tools=[...]
)
```
**When this happens:**
- User interrupts tool execution
- Client loses tool results due to network issues
- Conversation flow changes before tool completes
- Multi-turn conversations where tools are optional
### Case B: Orphaned Tool Results (Invalid tool_call_id)
**Problem:** A tool message references a `tool_call_id` that doesn't exist in any previous assistant message.
**Solution:** LiteLLM automatically removes these orphaned tool result messages.
**Example:**
```python
import litellm
litellm.modify_params = True
# Messages with orphaned tool result
messages = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi! How can I help?"},
{
"role": "tool",
"tool_call_id": "call_nonexistent", # This tool_call_id doesn't exist!
"content": "Some result"
}
]
# LiteLLM automatically removes the orphaned tool message
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages
)
```
**When this happens:**
- Message history is manually edited
- Tool results are duplicated or mismatched
- Conversation state is restored incorrectly
- Messages are merged from different conversations
### Case C: Empty Message Content
**Problem:** User or assistant messages have empty or whitespace-only content.
**Solution:** LiteLLM replaces empty content with a system placeholder message.
**Example:**
```python
import litellm
litellm.modify_params = True
# Messages with empty content
messages = [
{"role": "user", "content": ""}, # Empty content
{"role": "assistant", "content": " "}, # Whitespace only
]
# LiteLLM automatically replaces with:
# {"role": "user", "content": "[System: Empty message content sanitised to satisfy protocol]"}
# {"role": "assistant", "content": "[System: Empty message content sanitised to satisfy protocol]"}
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages
)
```
**When this happens:**
- UI sends empty messages
- Content is stripped during preprocessing
- Placeholder messages in conversation history
- Edge cases in message construction
## Configuration
### Enable Globally
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
# Enable for all completion calls
litellm.modify_params = True
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
litellm_settings:
modify_params: true
```
</TabItem>
<TabItem value="env" label="Environment Variable">
```bash
export LITELLM_MODIFY_PARAMS=True
```
</TabItem>
</Tabs>
### Enable Per-Request
```python
import litellm
# Enable only for specific requests
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages,
modify_params=True # Override global setting
)
```
## Supported Providers
Message sanitization currently works with:
- ✅ Anthropic (Claude)
**Note:** While the sanitization logic is provider-agnostic, it is currently only applied in the Anthropic message transformation pipeline. Support for additional providers may be added in future releases.
## Implementation Details
### How It Works
The message sanitization process runs **before** messages are converted to provider-specific formats:
1. **Input:** OpenAI-format messages with potential issues
2. **Sanitization:** Three helper functions process the messages:
- `_sanitize_empty_text_content()` - Fixes empty content
- `_add_missing_tool_results()` - Adds dummy tool results
- `_is_orphaned_tool_result()` - Identifies orphaned results
3. **Output:** Clean, provider-compatible messages
### Code Reference
The sanitization logic is implemented in:
- `litellm/litellm_core_utils/prompt_templates/factory.py`
- Function: `sanitize_messages_for_tool_calling()`
### Logging
When sanitization occurs, LiteLLM logs debug messages:
```python
import litellm
litellm.set_verbose = True # Enable debug logging
# You'll see logs like:
# "_add_missing_tool_results: Found 1 orphaned tool calls. Adding dummy tool results."
# "_is_orphaned_tool_result: Found orphaned tool result with tool_call_id=call_123"
# "_sanitize_empty_text_content: Replaced empty text content in user message"
```
## Best Practices
### 1. Enable for Production Workflows
```python
# Recommended for production
litellm.modify_params = True
# Ensures robust handling of edge cases
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages,
tools=tools
)
```
### 2. Preserve Tool Results When Possible
While sanitization handles missing tool results, it's better to provide actual results:
```python
# Good: Provide actual tool results
messages = [
{"role": "user", "content": "Search for Python"},
{"role": "assistant", "tool_calls": [...]},
{"role": "tool", "tool_call_id": "call_123", "content": "Actual search results"}
]
# Fallback: Sanitization adds dummy result if missing
messages = [
{"role": "user", "content": "Search for Python"},
{"role": "assistant", "tool_calls": [...]},
# Missing tool result - sanitization adds dummy
]
```
### 3. Monitor Sanitization Events
Use logging to track when sanitization occurs:
```python
import litellm
import logging
# Enable debug logging
litellm.set_verbose = True
logging.basicConfig(level=logging.DEBUG)
# Track sanitization events in your application
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=messages
)
```
### 4. Test Edge Cases
Ensure your application handles sanitized messages correctly:
```python
import litellm
litellm.modify_params = True
# Test orphaned tool calls
test_messages = [
{"role": "user", "content": "Test"},
{"role": "assistant", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "test", "arguments": "{}"}}]},
{"role": "user", "content": "Continue"} # No tool result
]
response = litellm.completion(
model="anthropic/claude-3-5-sonnet-20241022",
messages=test_messages,
tools=[...]
)
# Verify the response handles the dummy tool result appropriately
```
## Related Features
- **[Drop Params](./drop_params.md)** - Drop unsupported parameters for specific providers
- **[Message Trimming](./message_trimming.md)** - Trim messages to fit token limits
- **[Function Calling](./function_call.md)** - Complete guide to tool/function calling
- **[Reasoning Content](../reasoning_content.md)** - Extended thinking with tool calling
## Troubleshooting
### Sanitization Not Working
**Issue:** Messages still cause errors despite `modify_params=True`
**Solution:**
1. Verify `modify_params` is enabled:
```python
import litellm
print(litellm.modify_params) # Should be True
```
2. Check if the issue is provider-specific:
```python
litellm.set_verbose = True # Enable debug logging
```
3. Ensure you're using a recent version of LiteLLM:
```bash
pip install --upgrade litellm
```
### Unexpected Dummy Tool Results
**Issue:** Dummy tool results appear when you expect actual results
**Cause:** Tool result messages are missing or have incorrect `tool_call_id`
**Solution:**
1. Verify tool result messages have correct `tool_call_id`:
```python
# Correct
{"role": "tool", "tool_call_id": "call_123", "content": "result"}
# Incorrect - will be treated as orphaned
{"role": "tool", "tool_call_id": "wrong_id", "content": "result"}
```
2. Ensure tool results immediately follow assistant messages with tool_calls
### Performance Impact
**Issue:** Concerned about performance overhead
**Details:** Message sanitization has minimal performance impact:
- Runs in O(n) time where n = number of messages
- Only processes messages when `modify_params=True`
- Typically adds < 1ms to request processing time
## FAQ
**Q: Does sanitization modify my original messages?**
A: No, sanitization creates a new list of messages. Your original messages remain unchanged.
**Q: Can I disable specific sanitization cases?**
A: Currently, all three cases are handled together when `modify_params=True`. To disable sanitization entirely, set `modify_params=False`.
**Q: What happens to the dummy tool results?**
A: Dummy tool results are sent to the LLM provider along with other messages. The model sees them as regular tool results with informative error messages.
**Q: Does this work with streaming?**
A: Yes, message sanitization works with both streaming and non-streaming requests.
**Q: Is this related to `drop_params`?**
A: No, they're separate features:
- `modify_params` - Modifies/fixes message content and structure
- `drop_params` - Removes unsupported API parameters
Both can be enabled simultaneously.
## See Also
- [Reasoning Content with Tool Calling](../reasoning_content.md)
- [Function Calling Guide](./function_call.md)
- [Bedrock Provider Documentation](../providers/bedrock.md)
- [Anthropic Provider Documentation](../providers/anthropic.md)

48
docs/my-website/docs/completion/usage.md
View file

@ -50,3 +50,51 @@ for chunk in completion:
print(chunk.choices[0].delta)
```
### Proxy: Always Include Streaming Usage
When using the LiteLLM Proxy, you can configure it to automatically include usage information in all streaming responses, even if the client doesn't send `stream_options={"include_usage": True}`.
#### Configuration
Add the following to your config.yaml:
```yaml
general_settings:
always_include_stream_usage: true
```
Alternatively, configure it through the UI:
1. Navigate to the LiteLLM Proxy UI
2. Go to `Settings` > `Router Settings` > `General`
3. Find the `always_include_stream_usage` setting
4. Toggle it to `true`
5. Click `Update` to save
#### How it works
When `always_include_stream_usage` is enabled:
- All streaming requests will automatically have `stream_options={"include_usage": True}` added
- Clients will receive usage information in the final chunk, even if they didn't explicitly request it
- If a client already provides `stream_options`, `include_usage: True` will be added without overwriting other options
- Non-streaming requests are not affected
#### Example
With this setting enabled, a simple streaming request like:
```bash
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Hello!"}],
"stream": true
}'
```
Will automatically receive usage information in the response, without needing to explicitly include `stream_options`.
```

441
docs/my-website/docs/evals_api.md Normal file
View file

@ -0,0 +1,441 @@
# /evals
LiteLLM Proxy supports OpenAI's Evaluations (Evals) API, allowing you to create, manage, and run evaluations to measure model performance against defined testing criteria.
## What are Evals?
OpenAI Evals API provides a structured way to:
- **Create Evaluations**: Define testing criteria and data sources for evaluating model outputs
- **Run Evaluations**: Execute evaluations against specific models and datasets
- **Track Results**: Monitor evaluation progress and review detailed results
## Quick Start
### Setup LiteLLM Proxy
First, start your LiteLLM Proxy server:
```bash
litellm --config config.yaml
# Proxy will run on http://localhost:4000
```
### Initialize OpenAI Client
```python
from openai import OpenAI
# Point to your LiteLLM Proxy
client = OpenAI(
api_key="sk-1234", # Your LiteLLM proxy API key
base_url="http://localhost:4000" # Your proxy URL
)
```
For async operations:
```python
from openai import AsyncOpenAI
client = AsyncOpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
```
---
## Evaluation Management
### Create an Evaluation
Create an evaluation with testing criteria and data source configuration.
#### Example: Sentiment Classification Eval
```python
from openai import OpenAI
client = OpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
# Create evaluation with label model grader
eval_obj = client.evals.create(
name="Sentiment Classification",
data_source_config={
"type": "stored_completions",
"metadata": {"usecase": "chatbot"}
},
testing_criteria=[
{
"type": "label_model",
"model": "gpt-4o-mini",
"input": [
{
"role": "developer",
"content": "Classify the sentiment of the following statement as one of 'positive', 'neutral', or 'negative'"
},
{
"role": "user",
"content": "Statement: {{item.input}}"
}
],
"passing_labels": ["positive"],
"labels": ["positive", "neutral", "negative"],
"name": "Sentiment Grader"
}
]
)
# Note: If you want to use model-specific credentials for this evaluation, you can specify the model name in the extra body parameters.
print(f"Created eval: {eval_obj.id}")
print(f"Eval name: {eval_obj.name}")
```
#### Example: Push Notifications Summarizer Monitoring
This example shows how to monitor prompt changes for regressions in a push notifications summarizer:
```python
from openai import AsyncOpenAI
client = AsyncOpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
# Define data source for stored completions
data_source_config = {
"type": "stored_completions",
"metadata": {
"usecase": "push_notifications_summarizer"
}
}
# Define grader criteria
GRADER_DEVELOPER_PROMPT = """
Label the following push notification summary as either correct or incorrect.
The push notification and the summary will be provided below.
A good push notification summary is concise and snappy.
If it is good, then label it as correct, if not, then incorrect.
"""
GRADER_TEMPLATE_PROMPT = """
Push notifications: {{item.input}}
Summary: {{sample.output_text}}
"""
push_notification_grader = {
"name": "Push Notification Summary Grader",
"type": "label_model",
"model": "gpt-4o-mini",
"input": [
{
"role": "developer",
"content": GRADER_DEVELOPER_PROMPT,
},
{
"role": "user",
"content": GRADER_TEMPLATE_PROMPT,
},
],
"passing_labels": ["correct"],
"labels": ["correct", "incorrect"],
}
# Create the evaluation
eval_result = await client.evals.create(
name="Push Notification Completion Monitoring",
metadata={"description": "This eval monitors completions"},
data_source_config=data_source_config,
testing_criteria=[push_notification_grader],
)
eval_id = eval_result.id
print(f"Created eval: {eval_id}")
```
### List Evaluations
Retrieve a list of all your evaluations with pagination support.
```python
# List all evaluations
evals_response = client.evals.list(
limit=20,
order="desc"
)
for eval in evals_response.data:
print(f"Eval ID: {eval.id}, Name: {eval.name}")
# Check if there are more evals
if evals_response.has_more:
# Fetch next page
next_evals = client.evals.list(
after=evals_response.last_id,
limit=20
)
```
### Get a Specific Evaluation
Retrieve details of a specific evaluation by ID.
```python
eval = client.evals.retrieve(
eval_id="eval_abc123"
)
print(f"Eval ID: {eval.id}")
print(f"Name: {eval.name}")
print(f"Data Source: {eval.data_source_config}")
print(f"Testing Criteria: {eval.testing_criteria}")
```
### Update an Evaluation
Update evaluation metadata or name.
```python
updated_eval = client.evals.update(
eval_id="eval_abc123",
name="Updated Evaluation Name",
metadata={
"version": "2.0",
"updated_by": "user@example.com"
}
)
print(f"Updated eval: {updated_eval.name}")
```
### Delete an Evaluation
Permanently delete an evaluation.
```python
delete_response = client.evals.delete(
eval_id="eval_abc123"
)
print(f"Deleted: {delete_response.deleted}") # True
```
---
## Evaluation Runs
### Create a Run
Execute an evaluation by creating a run. The run processes your data through the model and applies testing criteria.
#### Using Stored Completions
First, generate some test data by making chat completions with metadata:
```python
from openai import AsyncOpenAI
import asyncio
client = AsyncOpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
# Generate test data with different prompt versions
push_notification_data = [
"""
- New message from Sarah: "Can you call me later?"
- Your package has been delivered!
- Flash sale: 20% off electronics for the next 2 hours!
""",
"""
- Weather alert: Thunderstorm expected in your area.
- Reminder: Doctor's appointment at 3 PM.
- John liked your photo on Instagram.
"""
]
PROMPTS = [
(
"""
You are a helpful assistant that summarizes push notifications.
You are given a list of push notifications and you need to collapse them into a single one.
Output only the final summary, nothing else.
""",
"v1"
),
(
"""
You are a helpful assistant that summarizes push notifications.
You are given a list of push notifications and you need to collapse them into a single one.
The summary should be longer than it needs to be and include more information than is necessary.
Output only the final summary, nothing else.
""",
"v2"
)
]
# Create completions with metadata for tracking
tasks = []
for notifications in push_notification_data:
for (prompt, version) in PROMPTS:
tasks.append(client.chat.completions.create(
model="gpt-4o-mini",
messages=[
{"role": "developer", "content": prompt},
{"role": "user", "content": notifications},
],
metadata={
"prompt_version": version,
"usecase": "push_notifications_summarizer"
}
))
await asyncio.gather(*tasks)
```
Now create runs to evaluate different prompt versions:
```python
# Grade prompt_version=v1
eval_run_result = await client.evals.runs.create(
eval_id=eval_id,
name="v1-run",
data_source={
"type": "completions",
"source": {
"type": "stored_completions",
"metadata": {
"prompt_version": "v1",
}
}
}
)
print(f"Run ID: {eval_run_result.id}")
print(f"Status: {eval_run_result.status}")
print(f"Report URL: {eval_run_result.report_url}")
# Grade prompt_version=v2
eval_run_result_v2 = await client.evals.runs.create(
eval_id=eval_id,
name="v2-run",
data_source={
"type": "completions",
"source": {
"type": "stored_completions",
"metadata": {
"prompt_version": "v2",
}
}
}
)
print(f"Run ID: {eval_run_result_v2.id}")
print(f"Report URL: {eval_run_result_v2.report_url}")
```
#### Using Completions with Different Models
Test how different models perform on the same inputs:
```python
# Test with GPT-4o using stored completions as input
tasks = []
for prompt_version in ["v1", "v2"]:
tasks.append(client.evals.runs.create(
eval_id=eval_id,
name=f"gpt-4o-run-{prompt_version}",
data_source={
"type": "completions",
"input_messages": {
"type": "item_reference",
"item_reference": "item.input",
},
"model": "gpt-4o",
"source": {
"type": "stored_completions",
"metadata": {
"prompt_version": prompt_version,
}
}
}
))
results = await asyncio.gather(*tasks)
for run in results:
print(f"Report URL: {run.report_url}")
```
### List Runs
Get all runs for a specific evaluation.
```python
# List all runs for an evaluation
runs_response = client.evals.runs.list(
eval_id="eval_abc123",
limit=20,
order="desc"
)
for run in runs_response.data:
print(f"Run ID: {run.id}")
print(f"Status: {run.status}")
print(f"Name: {run.name}")
if run.result_counts:
print(f"Results: {run.result_counts.passed}/{run.result_counts.total} passed")
```
### Get Run Details
Retrieve detailed information about a specific run, including results.
```python
run = client.evals.runs.retrieve(
eval_id="eval_abc123",
run_id="run_def456"
)
print(f"Run ID: {run.id}")
print(f"Status: {run.status}")
print(f"Started: {run.started_at}")
print(f"Completed: {run.completed_at}")
# Check results
if run.result_counts:
print(f"\nOverall Results:")
print(f"Total: {run.result_counts.total}")
print(f"Passed: {run.result_counts.passed}")
print(f"Failed: {run.result_counts.failed}")
print(f"Error: {run.result_counts.errored}")
# Per-criteria results
if run.per_testing_criteria_results:
for criteria_result in run.per_testing_criteria_results:
print(f"\nCriteria {criteria_result.testing_criteria_index}:")
print(f" Passed: {criteria_result.result_counts.passed}")
print(f" Average Score: {criteria_result.average_score}")
```
### Delete a Run
Permanently delete a run and its results.
```python
delete_response = await client.evals.runs.delete(
eval_id="eval_abc123",
run_id="run_def456"
)
print(f"Deleted: {delete_response.deleted}") # True
print(f"Run ID: {delete_response.run_id}")
```

62
docs/my-website/docs/mcp.md
View file

@ -808,6 +808,68 @@ If your stdio MCP server needs per-request credentials, you can map HTTP headers
In this example, when a client makes a request with the `X-GITHUB_PERSONAL_ACCESS_TOKEN` header, the proxy forwards that value into the stdio process as the `GITHUB_PERSONAL_ACCESS_TOKEN` environment variable.
## Control MCP Access for End Users
Control which MCP servers end users of your AI application can access (e.g. users of an internal chat UI). Pass the customer ID in the `x-litellm-end-user-id` header to:
- Enforce object permissions (limit which MCP servers they can access)
- Apply customer-specific budgets
- Track spend per customer
**FastMCP Client Example:**
```python title="Track customer spend with x-litellm-end-user-id" showLineNumbers
from fastmcp import Client
import asyncio
# MCP client configuration with customer tracking
config = {
"mcpServers": {
"github": {
"url": "http://localhost:4000/github_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234",
"x-litellm-end-user-id": "customer_123", # 👈 CUSTOMER ID
"Authorization": "Bearer gho_token"
}
}
}
}
client = Client(config)
async def main():
async with client:
# All MCP calls will be tracked under customer_123
tools = await client.list_tools()
result = await client.call_tool(tools[0].name, {})
print(f"Tool result: {result}")
asyncio.run(main())
```
**Cursor IDE Example:**
```json title="Cursor config with customer tracking" showLineNumbers
{
"mcpServers": {
"GitHub": {
"url": "http://localhost:4000/github_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY",
"x-litellm-end-user-id": "customer_123"
}
}
}
}
```
**What happens:**
- Customer-specific object permissions are enforced (only allowed MCP servers are accessible)
- Customer budgets are applied
- All tool calls are tracked under `customer_123`
[Learn more about customer management →](./proxy/customers)
## Using your MCP with client side credentials
Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.

9
docs/my-website/docs/observability/datadog.md
View file

@ -253,3 +253,12 @@ LiteLLM supports customizing the following Datadog environment variables
\* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required
\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required for **Datadog Logs**. (**Note: `DD_API_KEY` IS REQUIRED for Datadog LLM Observability**)
## Automatic Tags
LiteLLM automatically adds the following tags to your Datadog logs and metrics if the information is available in the request:
| Tag | Description | Source |
|-----|-------------|--------|
| `team` | The team alias or ID associated with the API Key | `user_api_key_team_alias`, `team_alias`, `user_api_key_team_id`, or `team_id` in metadata |
| `request_tag` | Custom tags passed in the request | `request_tags` in logging payload |

113
docs/my-website/docs/projects/openai-agents.md
View file

@ -1,22 +1,121 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# OpenAI Agents SDK
The [OpenAI Agents SDK](https://github.com/openai/openai-agents-python) is a lightweight framework for building multi-agent workflows.
It includes an official LiteLLM extension that lets you use any of the 100+ supported providers (Anthropic, Gemini, Mistral, Bedrock, etc.)
Use OpenAI Agents SDK with any LLM provider through LiteLLM Proxy.
The [OpenAI Agents SDK](https://github.com/openai/openai-agents-python) is a lightweight framework for building multi-agent workflows. It includes an official LiteLLM extension that lets you use any of the 100+ supported providers.
## Quick Start
### 1. Install Dependencies
```bash
pip install "openai-agents[litellm]"
```
### 2. Add Model to Config
```yaml title="config.yaml"
model_list:
- model_name: gpt-4o
litellm_params:
model: "openai/gpt-4o"
api_key: "os.environ/OPENAI_API_KEY"
- model_name: claude-sonnet
litellm_params:
model: "anthropic/claude-3-5-sonnet-20241022"
api_key: "os.environ/ANTHROPIC_API_KEY"
- model_name: gemini-pro
litellm_params:
model: "gemini/gemini-2.0-flash-exp"
api_key: "os.environ/GEMINI_API_KEY"
```
### 3. Start LiteLLM Proxy
```bash
litellm --config config.yaml
```
### 4. Use with Proxy
<Tabs>
<TabItem value="proxy" label="Via Proxy">
```python
from agents import Agent, Runner
from agents.extensions.models.litellm_model import LitellmModel
# Point to LiteLLM proxy
agent = Agent(
name="Assistant",
instructions="You are a helpful assistant.",
model=LitellmModel(model="provider/model-name")
model=LitellmModel(
model="claude-sonnet", # Model from config.yaml
api_key="sk-1234", # LiteLLM API key
base_url="http://localhost:4000"
)
)
result = Runner.run_sync(agent, "your_prompt_here")
print("Result:", result.final_output)
result = await Runner.run(agent, "What is LiteLLM?")
print(result.final_output)
```
- [GitHub](https://github.com/openai/openai-agents-python)
- [LiteLLM Extension Docs](https://openai.github.io/openai-agents-python/ref/extensions/litellm/)
</TabItem>
<TabItem value="direct" label="Direct (No Proxy)">
```python
from agents import Agent, Runner
from agents.extensions.models.litellm_model import LitellmModel
# Use any provider directly
agent = Agent(
name="Assistant",
instructions="You are a helpful assistant.",
model=LitellmModel(
model="anthropic/claude-3-5-sonnet-20241022",
api_key="your-anthropic-key"
)
)
result = await Runner.run(agent, "What is LiteLLM?")
print(result.final_output)
```
</TabItem>
</Tabs>
## Track Usage
Enable usage tracking to monitor token consumption:
```python
from agents import Agent, ModelSettings
from agents.extensions.models.litellm_model import LitellmModel
agent = Agent(
name="Assistant",
model=LitellmModel(model="claude-sonnet", api_key="sk-1234"),
model_settings=ModelSettings(include_usage=True)
)
result = await Runner.run(agent, "Hello")
print(result.context_wrapper.usage) # Token counts
```
## Environment Variables
| Variable | Value | Description |
|----------|-------|-------------|
| `LITELLM_BASE_URL` | `http://localhost:4000` | LiteLLM proxy URL |
| `LITELLM_API_KEY` | `sk-1234` | Your LiteLLM API key |
## Related Resources
- [OpenAI Agents SDK Documentation](https://openai.github.io/openai-agents-python/)
- [LiteLLM Extension Docs](https://openai.github.io/openai-agents-python/models/litellm/)
- [LiteLLM Proxy Quick Start](../proxy/quick_start)

2
docs/my-website/docs/providers/gemini.md
View file

@ -1196,6 +1196,8 @@ When responding to Computer Use tool calls, include the URL and screenshot:
## Thought Signatures
Thought signatures are encrypted representations of the model's internal reasoning process for a given turn in a conversation. By passing thought signatures back to the model in subsequent requests, you provide it with the context of its previous thoughts, allowing it to build upon its reasoning and maintain a coherent line of inquiry.

52
docs/my-website/docs/providers/watsonx/rerank.md Normal file
View file

@ -0,0 +1,52 @@
# watsonx.ai Rerank
## Overview
| Property | Details |
|----------|--------------------------------------------------------------------------|
| Description | watsonx.ai rerank integration |
| Provider Route on LiteLLM | `watsonx/` |
| Supported Operations | `/ml/v1/text/rerank` |
| Link to Provider Doc | [IBM WatsonX.ai ↗](https://cloud.ibm.com/apidocs/watsonx-ai#text-rerank) |
## Quick Start
### **LiteLLM SDK**
```python
import os
from litellm import rerank
os.environ["WATSONX_APIKEY"] = "YOUR_WATSONX_APIKEY"
os.environ["WATSONX_API_BASE"] = "YOUR_WATSONX_API_BASE"
os.environ["WATSONX_PROJECT_ID"] = "YOUR_WATSONX_PROJECT_ID"
query="Best programming language for beginners?"
documents=[
"Python is great for beginners due to simple syntax.",
"JavaScript runs in browsers and is versatile.",
"Rust has a steep learning curve but is very safe.",
]
response = rerank(
model="watsonx/cross-encoder/ms-marco-minilm-l-12-v2",
query=query,
documents=documents,
top_n=2,
return_documents=True,
)
print(response)
```
### **LiteLLM Proxy**
```yaml
model_list:
- model_name: cross-encoder/ms-marco-minilm-l-12-v2
litellm_params:
model: watsonx/cross-encoder/ms-marco-minilm-l-12-v2
api_key: os.environ/WATSONX_APIKEY
api_base: os.environ/WATSONX_API_BASE
project_id: os.environ/WATSONX_PROJECT_ID
```

9
docs/my-website/docs/proxy/config_settings.md
View file

@ -358,7 +358,8 @@ router_settings:
| redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
| cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. |
| router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) |
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Currently supported: 'router_budget_limiting', 'prompt_caching' |
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `deployment_affinity`, `forward_client_headers_by_model_group` |
| deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). |
| ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. |
| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) |
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
@ -450,6 +451,7 @@ router_settings:
| BATCH_STATUS_POLL_INTERVAL_SECONDS | Interval in seconds for polling batch status. Default is 3600 (1 hour)
| BATCH_STATUS_POLL_MAX_ATTEMPTS | Maximum number of attempts for polling batch status. Default is 24 (for 24 hours)
| BEDROCK_MAX_POLICY_SIZE | Maximum size for Bedrock policy. Default is 75
| BEDROCK_MIN_THINKING_BUDGET_TOKENS | Minimum thinking budget in tokens for Bedrock reasoning models. Bedrock returns a 400 error if budget_tokens is below this value. Requests with lower values are clamped to this minimum. Default is 1024
| BERRISPEND_ACCOUNT_ID | Account ID for BerriSpend service
| BRAINTRUST_API_KEY | API key for Braintrust integration
| BRAINTRUST_API_BASE | Base URL for Braintrust API. Default is https://api.braintrustdata.com/v1
@ -492,6 +494,7 @@ router_settings:
| DATABASE_USER | Username for database connection
| DATABASE_USERNAME | Alias for database user
| DATABRICKS_API_BASE | Base URL for Databricks API
| DATABRICKS_API_KEY | API key (Personal Access Token) for Databricks API authentication
| DATABRICKS_CLIENT_ID | Client ID for Databricks OAuth M2M authentication (Service Principal application ID)
| DATABRICKS_CLIENT_SECRET | Client secret for Databricks OAuth M2M authentication
| DATABRICKS_USER_AGENT | Custom user agent string for Databricks API requests. Used for partner telemetry attribution
@ -539,7 +542,7 @@ router_settings:
| DEFAULT_IMAGE_WIDTH | Default width for images. Default is 300
| DEFAULT_IN_MEMORY_TTL | Default time-to-live for in-memory cache in seconds. Default is 5
| DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL | Default time-to-live in seconds for management objects (User, Team, Key, Organization) in memory cache. Default is 60 seconds.
| DEFAULT_MAX_LRU_CACHE_SIZE | Default maximum size for LRU cache. Default is 16
| DEFAULT_MAX_LRU_CACHE_SIZE | Default maximum size for LRU cache. Default is 64
| DEFAULT_MAX_RECURSE_DEPTH | Default maximum recursion depth. Default is 100
| DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER | Default maximum recursion depth for sensitive data masker. Default is 10
| DEFAULT_MAX_RETRIES | Default maximum retry attempts. Default is 2
@ -602,7 +605,6 @@ router_settings:
| EMAIL_BUDGET_ALERT_TTL | Time-to-live for budget alert deduplication in seconds. Default is 86400 (24 hours)
| ENKRYPTAI_API_BASE | Base URL for EnkryptAI Guardrails API. **Default is https://api.enkryptai.com**
| ENKRYPTAI_API_KEY | API key for EnkryptAI Guardrails service
| EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False**
| FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4
| FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16
| FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56
@ -769,6 +771,7 @@ router_settings:
| LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval.
| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
| LITELLM_KEY_ROTATION_GRACE_PERIOD | Duration to keep old key valid after rotation (e.g. "24h", "2d"). Default is empty (immediate revoke). Used for scheduled rotations and as fallback when not specified in regenerate request.
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False`
| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM

263
docs/my-website/docs/proxy/customers.md
View file

@ -2,29 +2,98 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Customers / End-User Budgets
# Customers / End-Users
Track spend, set budgets for your customers.
Track spend, set budgets and permissions for your customers.
## Tracking Customer Spend
## Tracking Customer Spend + Permissions
### 1. Make LLM API call w/ Customer ID
Make a /chat/completions call, pass 'user' - First call Works
LiteLLM checks for a customer/end-user ID in the following order (first match wins):
```bash showLineNumbers title="Make request with customer ID"
| Priority | Method | Where | Notes |
|----------|--------|-------|-------|
| 1 | `x-litellm-customer-id` header | Request headers | Standard header, always checked |
| 2 | `x-litellm-end-user-id` header | Request headers | Standard header, always checked |
| 3 | Custom header via `user_header_mappings` | Request headers | Configured in `general_settings` |
| 4 | Custom header via `user_header_name` | Request headers | Deprecated — use `user_header_mappings` |
| 5 | `user` field | Request body | Standard OpenAI field |
| 6 | `litellm_metadata.user` field | Request body | Anthropic-style metadata |
| 7 | `metadata.user_id` field | Request body | Generic metadata pattern |
| 8 | `safety_identifier` field | Request body | Responses API |
**Option 1: Standard headers** (recommended — no request body modification needed)
```bash showLineNumbers title="Make request with customer ID in header"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY
--data ' {
--header 'Authorization: Bearer sk-1234' \
--header 'x-litellm-end-user-id: ishaan3' \
--data '{
"model": "azure-gpt-3.5",
"user": "ishaan3", # 👈 CUSTOMER ID
"messages": [
{
"role": "user",
"content": "what time is it"
}
]
"messages": [{"role": "user", "content": "what time is it"}]
}'
```
Both `x-litellm-customer-id` and `x-litellm-end-user-id` are supported and always checked without any configuration.
**Option 2: `user` field in request body** (OpenAI-compatible)
```bash showLineNumbers title="Make request with customer ID in body"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "azure-gpt-3.5",
"user": "ishaan3",
"messages": [{"role": "user", "content": "what time is it"}]
}'
```
**Option 3: Custom header via `user_header_mappings`** (configurable)
```yaml showLineNumbers title="config.yaml"
general_settings:
user_header_mappings:
- header_name: "x-my-app-user-id"
litellm_user_role: "customer"
```
```bash showLineNumbers title="Make request with custom header"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--header 'x-my-app-user-id: ishaan3' \
--data '{
"model": "azure-gpt-3.5",
"messages": [{"role": "user", "content": "what time is it"}]
}'
```
**Option 4: `litellm_metadata.user`** (Anthropic-style)
```bash showLineNumbers title="Make request with litellm_metadata.user"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "claude-3-5-sonnet",
"messages": [{"role": "user", "content": "what time is it"}],
"litellm_metadata": {"user": "ishaan3"}
}'
```
**Option 5: `metadata.user_id`**
```bash showLineNumbers title="Make request with metadata.user_id"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "azure-gpt-3.5",
"messages": [{"role": "user", "content": "what time is it"}],
"metadata": {"user_id": "ishaan3"}
}'
```
@ -123,7 +192,171 @@ Expected Response
</Tabs>
## Setting Customer Budgets
## Setting Customer Object Permissions
Control which resources (MCP servers, vector stores, agents) a customer can access.
### What are Object Permissions?
Object permissions allow you to restrict customer access to specific:
- **MCP Servers**: Limit which MCP servers the customer can call
- **MCP Access Groups**: Assign customers to predefined groups of MCP servers
- **MCP Tool Permissions**: Granular control over which tools within an MCP server the customer can use
- **Vector Stores**: Control which vector stores the customer can query
- **Agents**: Restrict which agents the customer can interact with
- **Agent Access Groups**: Assign customers to predefined groups of agents
### Creating a Customer with Object Permissions
```bash showLineNumbers title="Create customer with object permissions"
curl -L -X POST 'http://localhost:4000/customer/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "user_1",
"object_permission": {
"mcp_servers": ["server_1", "server_2"],
"mcp_access_groups": ["public_group"],
"mcp_tool_permissions": {
"server_1": ["tool_a", "tool_b"]
},
"vector_stores": ["vector_store_1"],
"agents": ["agent_1"],
"agent_access_groups": ["basic_agents"]
}
}'
```
**Parameters:**
- `mcp_servers` (Optional[List[str]]): List of allowed MCP server IDs
- `mcp_access_groups` (Optional[List[str]]): List of MCP access group names
- `mcp_tool_permissions` (Optional[Dict[str, List[str]]]): Map of server ID to allowed tool names
- `vector_stores` (Optional[List[str]]): List of allowed vector store IDs
- `agents` (Optional[List[str]]): List of allowed agent IDs
- `agent_access_groups` (Optional[List[str]]): List of agent access group names
**Note:** If `object_permission` is `null` or `{}`, the customer has no object-level restrictions.
### Updating Customer Object Permissions
You can update object permissions for existing customers:
```bash showLineNumbers title="Update customer object permissions"
curl -L -X POST 'http://localhost:4000/customer/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "user_1",
"object_permission": {
"mcp_servers": ["server_3"],
"vector_stores": ["vector_store_2", "vector_store_3"]
}
}'
```
### Viewing Customer Object Permissions
When you query customer info, object permissions are included in the response:
```bash showLineNumbers title="Get customer info with object permissions"
curl -X GET 'http://0.0.0.0:4000/customer/info?end_user_id=user_1' \
-H 'Authorization: Bearer sk-1234'
```
**Response:**
```json showLineNumbers title="Response with object permissions"
{
"user_id": "user_1",
"blocked": false,
"alias": "John Doe",
"spend": 0.0,
"object_permission": {
"object_permission_id": "perm_abc123",
"mcp_servers": ["server_1", "server_2"],
"mcp_access_groups": ["public_group"],
"mcp_tool_permissions": {
"server_1": ["tool_a", "tool_b"]
},
"vector_stores": ["vector_store_1"],
"agents": ["agent_1"],
"agent_access_groups": ["basic_agents"]
},
"litellm_budget_table": null
}
```
### Use Cases
**1. Tiered Access Control**
Create different permission tiers for your customers:
```bash showLineNumbers title="Free tier customer"
# Free tier - limited access
curl -L -X POST 'http://localhost:4000/customer/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "free_user",
"budget_id": "free_tier",
"object_permission": {
"mcp_access_groups": ["public_group"],
"agent_access_groups": ["basic_agents"]
}
}'
```
```bash showLineNumbers title="Premium tier customer"
# Premium tier - full access
curl -L -X POST 'http://localhost:4000/customer/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "premium_user",
"budget_id": "premium_tier",
"object_permission": {
"mcp_servers": ["server_1", "server_2", "server_3"],
"vector_stores": ["vector_store_1", "vector_store_2"],
"agents": ["agent_1", "agent_2", "agent_3"]
}
}'
```
**2. Department-Specific Access**
Restrict customers to resources relevant to their department:
```bash showLineNumbers title="Sales team customer"
curl -L -X POST 'http://localhost:4000/customer/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "sales_user",
"object_permission": {
"mcp_servers": ["crm_server", "email_server"],
"agents": ["sales_assistant"],
"vector_stores": ["sales_knowledge_base"]
}
}'
```
**3. Tool-Level Restrictions**
Grant access to specific tools within an MCP server:
```bash showLineNumbers title="Limited tool access"
curl -L -X POST 'http://localhost:4000/customer/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"user_id": "restricted_user",
"object_permission": {
"mcp_servers": ["database_server"],
"mcp_tool_permissions": {
"database_server": ["read_only_query", "get_table_schema"]
}
}
}'
```
## Setting Customer Budgets
Set customer budgets (e.g. monthly budgets, tpm/rpm limits) on LiteLLM Proxy

1
docs/my-website/docs/proxy/logging.md
View file

@ -1338,6 +1338,7 @@ litellm_settings:
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets
s3_use_virtual_hosted_style: false # [OPTIONAL] use virtual-hosted-style URLs (bucket.endpoint/key) instead of path-style (endpoint/bucket/key). Useful for S3-compatible services like MinIO
s3_strip_base64_files: false # [OPTIONAL] remove base64 files before storing in s3
```

111
docs/my-website/docs/proxy/pass_through.md
View file

@ -58,6 +58,17 @@ Configure the required authentication and pricing:
- The Bria API requires an `api_token` header
- Enter your Bria API key as the value for the `api_token` header
**Default Query Parameters (Optional):**
- Add query parameters that will be automatically sent with every request
- Perfect for API versioning, format specifications, or default configurations
- Clients can override these parameters by providing their own values
- Example: `version=v1`, `format=json`, `timeout=30`
<Image
img={require('../../img/passthrough_query_default.png')}
style={{width: '60%', display: 'block', margin: '2rem auto'}}
/>
**Pricing Configuration:**
- Set a cost per request (e.g., $12.00 in this example)
- This enables cost tracking and billing for your users
@ -112,6 +123,9 @@ general_settings:
content-type: application/json
accept: application/json
forward_headers: true # Forward all incoming headers
default_query_params: # Optional: Default query parameters
version: "v1" # Always send version=v1
format: "json" # Default format (can be overridden)
```
### Start and Test
@ -166,6 +180,9 @@ general_settings:
auth: boolean # Enable LiteLLM authentication (Enterprise)
forward_headers: boolean # Forward all incoming headers
include_subpath: boolean # If true, forwards requests to sub-paths (default: false)
methods: list[string] # Optional: HTTP methods (e.g., ["GET", "POST"]). If not specified, all methods are supported.
default_query_params: # Optional: Default query parameters sent with every request
<param-name>: string # Key-value pairs (e.g., version: "v1", format: "json")
headers: # Custom headers to add
Authorization: string # Auth header for target API
content-type: string # Request content type
@ -177,11 +194,17 @@ general_settings:
### Header Options
- **Authorization**: Authentication for the target API
- **content-type**: Request body format specification
- **content-type**: Request body format specification
- **accept**: Expected response format
- **LANGFUSE_PUBLIC_KEY/SECRET_KEY**: For Langfuse integration
- **Custom headers**: Any additional key-value pairs
### Default Query Parameters
- **Parameter precedence**: Client params > URL params > default params
- **Use cases**: API versioning, authentication tokens, format control, feature flags
- **Override capability**: Clients can override any default parameter
- **Examples**: `version: "v1"`, `format: "json"`, `timeout: "30"`
### Sub-path Routing
By default, pass-through endpoints only match the **exact path** specified. To forward requests to sub-paths, set `include_subpath: true`:
@ -201,6 +224,92 @@ general_settings:
---
### Default Query Parameters
Pass-through endpoints support default query parameters that are automatically added to every request. This is useful for API versioning, format specifications, authentication tokens, or any default configuration.
#### How It Works
**Parameter Precedence (highest to lowest priority):**
1. **Client-provided parameters** (in the request URL)
2. **URL parameters** (from the target URL)
3. **Default parameters** (from configuration)
#### Example Configuration
```yaml
general_settings:
pass_through_endpoints:
- path: "/api/v1"
target: "https://external-api.com/service?timeout=60" # URL has timeout=60
default_query_params:
version: "v1" # Always add version=v1
format: "json" # Default format=json (can be overridden)
auth_level: "basic" # Always add auth_level=basic
```
#### Request Examples
**Client Request:** `GET /api/v1/users`
**Actual Backend Call:** `https://external-api.com/service?version=v1&format=json&auth_level=basic&timeout=60`
**Client Request:** `GET /api/v1/users?format=xml&custom=value`
**Actual Backend Call:** `https://external-api.com/service?version=v1&auth_level=basic&timeout=60&format=xml&custom=value`
- Client `format=xml` overrides default `format=json`
- Default `version=v1` and `auth_level=basic` are preserved
- URL `timeout=60` is preserved
- Client `custom=value` is added
#### Use Cases
- **API Versioning**: Always send `version=v2` to maintain compatibility
- **Authentication**: Add authentication tokens like `api_key=default_key`
- **Format Control**: Default to `format=json` but allow client override
- **Rate Limiting**: Set `rate_limit=standard` as default
- **Feature Flags**: Enable `experimental=false` by default
---
You can configure different target URLs for the same path using different HTTP methods. This is useful when different backends handle different operations:
<Image
img={require('../../img/passthrough_method_setup.png')}
style={{width: '60%', display: 'block', margin: '2rem auto'}}
/>
```yaml
general_settings:
pass_through_endpoints:
# GET requests to /azure/kb go to read API
- path: "/azure/kb"
target: "https://read-api.example.com/knowledge-base"
methods: ["GET"]
headers:
Authorization: "bearer os.environ/READ_API_KEY"
# POST requests to /azure/kb go to write API
- path: "/azure/kb"
target: "https://write-api.example.com/knowledge-base"
methods: ["POST"]
headers:
Authorization: "bearer os.environ/WRITE_API_KEY"
# PUT requests to /azure/kb go to update API
- path: "/azure/kb"
target: "https://update-api.example.com/knowledge-base"
methods: ["PUT"]
headers:
Authorization: "bearer os.environ/UPDATE_API_KEY"
```
**Key Points:**
- If `methods` is not specified, the endpoint supports all HTTP methods (GET, POST, PUT, DELETE, PATCH)
- Multiple endpoints can share the same path as long as they have different methods
- You can specify multiple methods for a single endpoint: `methods: ["GET", "POST"]`
- This allows you to route to different backends based on the operation type
---
## Advanced: Custom Adapters
For complex integrations (like Anthropic/Bedrock clients), you can create custom adapters that translate between different API schemas.

318
docs/my-website/docs/proxy/project_management.md Normal file
View file

@ -0,0 +1,318 @@
# [Beta] Project Management
Projects in LiteLLM sit between teams and keys in the organizational hierarchy, enabling fine-grained access control and budget management for specific use cases or applications.
```mermaid
graph TD
A[Organization] --> B[Team 1]
A --> C[Team 2]
B --> D[Project A]
B --> E[Project B]
C --> F[Project C]
D --> G[API Key 1]
D --> H[API Key 2]
E --> I[API Key 3]
F --> J[API Key 4]
style A fill:#e1f5ff
style B fill:#fff4e6
style C fill:#fff4e6
style D fill:#f3e5f5
style E fill:#f3e5f5
style F fill:#f3e5f5
style G fill:#e8f5e9
style H fill:#e8f5e9
style I fill:#e8f5e9
style J fill:#e8f5e9
```
**Hierarchy**: `Organizations > Teams > Projects > Keys`
## Quick Start
This walkthrough shows how to create a project, generate an API key, make requests, and view project-level spend tracking in the UI.
### Step 1: Create a Project
```bash showLineNumbers
curl --location 'http://0.0.0.0:4000/project/new' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"project_alias": "flight-search-assistant",
"team_id": "ad898803-c8a3-4f4a-976a-a3c372cffa45",
"models": ["gpt-4", "gpt-3.5-turbo"],
"max_budget": 100,
"metadata": {
"use_case_id": "SNOW-12345",
"responsible_ai_id": "RAI-67890"
}
}' | jq
```
**Response:**
```json
{
"project_id": "e402a141-725a-4437-bff5-d47459189716",
"project_alias": "flight-search-assistant",
"team_id": "ad898803-c8a3-4f4a-976a-a3c372cffa45",
"models": ["gpt-4", "gpt-3.5-turbo"],
"max_budget": 100,
...
}
```
### Step 2: Generate API Key for Project
```bash showLineNumbers
curl 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data-raw '{
"models": ["gpt-3.5-turbo", "gpt-4"],
"metadata": {"user": "ishaan@berri.ai"},
"project_id": "e402a141-725a-4437-bff5-d47459189716"
}' | jq
```
**Response:**
```json
{
"key": "sk-W8VbscpfuyvHm5TkxRYiXA",
"key_name": "sk-...YiXA",
"project_id": "e402a141-725a-4437-bff5-d47459189716",
...
}
```
### Step 3: Use API Key in Chat Completions
```bash showLineNumbers
curl http://localhost:4000/v1/chat/completions \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-W8VbscpfuyvHm5TkxRYiXA' \
--data '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "What is litellm?"}]
}' | jq
```
### Step 4: View Project Spend in UI
Navigate to the **Logs** page in the LiteLLM Admin UI. You'll see the `user_api_key_project_id` tracked in the request metadata:
![Project Spend Tracking](/img/project_spend.png)
As shown above, the spend logs metadata includes:
- `"user_api_key_project_id": "e402a141-725a-4437-bff5-d47459189716"` - Links the request to your project
- All costs and token usage are automatically attributed to the project
- You can query and filter logs by project ID for detailed reporting
## API Endpoints
### POST /project/new
Create a new project.
**Who can call**: Admins or Team Admins
**Parameters**:
- `project_alias` (string, optional): Human-readable name for the project
- `team_id` (string, required): The team this project belongs to
- `models` (array, optional): List of models the project can access
- `max_budget` (float, optional): Maximum spend budget for the project
- `tpm_limit` (int, optional): Tokens per minute limit
- `rpm_limit` (int, optional): Requests per minute limit
- `budget_duration` (string, optional): Budget reset period (e.g., "30d", "1mo")
- `metadata` (object, optional): Custom metadata for the project
- `blocked` (boolean, optional): Block all API calls for this project
**Example**:
```bash
curl --location 'http://0.0.0.0:4000/project/new' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"project_alias": "hotel-recommendations",
"team_id": "team-123",
"models": ["claude-3-sonnet"],
"max_budget": 200,
"tpm_limit": 100000,
"metadata": {
"use_case_id": "SNOW-12346",
"cost_center": "travel-products"
}
}'
```
**Response**:
```json
{
"project_id": "project-def",
"project_alias": "hotel-recommendations",
"team_id": "team-123",
"models": ["claude-3-sonnet"],
"spend": 0.0,
"budget_id": "budget-xyz",
"metadata": {
"use_case_id": "SNOW-12346",
"cost_center": "travel-products"
},
"created_at": "2025-01-15T10:00:00Z",
"updated_at": "2025-01-15T10:00:00Z"
}
```
### POST /project/update
Update an existing project.
**Who can call**: Admins or Team Admins
**Parameters**:
- `project_id` (string, required): The project to update
- `project_alias` (string, optional): Updated project name
- `team_id` (string, optional): Move project to different team
- `models` (array, optional): Updated list of allowed models
- `max_budget` (float, optional): Updated budget
- `tpm_limit` (int, optional): Updated TPM limit
- `rpm_limit` (int, optional): Updated RPM limit
- `metadata` (object, optional): Updated metadata
- `blocked` (boolean, optional): Updated blocked status
**Example**:
```bash
curl --location 'http://0.0.0.0:4000/project/update' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"project_id": "project-abc",
"max_budget": 200,
"tpm_limit": 200000,
"metadata": {
"status": "production"
}
}'
```
### GET /project/info
Get information about a specific project.
**Parameters**:
- `project_id` (string, required): Query parameter
**Example**:
```bash
curl --location 'http://0.0.0.0:4000/project/info?project_id=project-abc' \
--header 'Authorization: Bearer sk-1234'
```
**Response**:
```json
{
"project_id": "project-abc",
"project_alias": "flight-search-assistant",
"team_id": "team-123",
"models": ["gpt-4", "gpt-3.5-turbo"],
"spend": 45.67,
"model_spend": {
"gpt-4": 42.30,
"gpt-3.5-turbo": 3.37
},
"litellm_budget_table": {
"budget_id": "budget-xyz",
"max_budget": 100.0,
"tpm_limit": 100000,
"rpm_limit": 100
},
"metadata": {
"use_case_id": "SNOW-12345"
}
}
```
### GET /project/list
List all projects the user has access to.
**Example**:
```bash
curl --location 'http://0.0.0.0:4000/project/list' \
--header 'Authorization: Bearer sk-1234'
```
**Response**:
```json
[
{
"project_id": "project-abc",
"project_alias": "flight-search-assistant",
"team_id": "team-123",
"spend": 45.67
},
{
"project_id": "project-def",
"project_alias": "hotel-recommendations",
"team_id": "team-123",
"spend": 23.45
}
]
```
### DELETE /project/delete
Delete one or more projects.
**Who can call**: Admins only
**Parameters**:
- `project_ids` (array, required): List of project IDs to delete
**Example**:
```bash
curl --location --request DELETE 'http://0.0.0.0:4000/project/delete' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"project_ids": ["project-abc", "project-def"]
}'
```
**Note**: Projects with associated API keys cannot be deleted. Delete or reassign the keys first.
## Model-Specific Quotas
You can set different quotas for different models within a project:
```bash
curl --location 'http://0.0.0.0:4000/project/new' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"project_alias": "multi-model-project",
"team_id": "team-123",
"models": ["gpt-4", "gpt-3.5-turbo", "claude-3-sonnet"],
"max_budget": 500,
"metadata": {
"model_tpm_limit": {
"gpt-4": 50000,
"gpt-3.5-turbo": 200000,
"claude-3-sonnet": 100000
},
"model_rpm_limit": {
"gpt-4": 50,
"gpt-3.5-turbo": 500,
"claude-3-sonnet": 100
}
}
}'
```

55
docs/my-website/docs/proxy/prompt_management.md
View file

@ -11,6 +11,7 @@ Run experiments or change the specific model (e.g. from gpt-4o to gpt4o-mini fin
| Native LiteLLM GitOps (.prompt files) | [Get Started](native_litellm_prompt) |
| Langfuse | [Get Started](https://langfuse.com/docs/prompts/get-started) |
| Humanloop | [Get Started](../observability/humanloop) |
| Generic Prompt Management API | [Get Started](../adding_provider/generic_prompt_management_api) |
## Onboarding Prompts via config.yaml
@ -34,7 +35,7 @@ prompts:
- prompt_id: "my_prompt_id"
litellm_params:
prompt_id: "my_prompt_id"
prompt_integration: "dotprompt" # or langfuse, bitbucket, gitlab, custom
prompt_integration: "dotprompt" # or langfuse, bitbucket, gitlab, generic_prompt_management, custom
# integration-specific parameters below
```
@ -46,6 +47,7 @@ The `prompt_integration` field determines where and how prompts are loaded:
- **`langfuse`**: Fetch prompts from Langfuse prompt management
- **`bitbucket`**: Load from BitBucket repository `.prompt` files (team-based access control)
- **`gitlab`**: Load from GitLab repository `.prompt` files (team-based access control)
- **`generic_prompt_management`**: Integrate any prompt management system via a simple API endpoint (no PR required)
- **`custom`**: Use your own custom prompt management implementation
Each integration has its own configuration parameters and access control mechanisms.
@ -207,6 +209,57 @@ System: You are a helpful assistant.
User: {{user_message}}
```
</TabItem>
<TabItem value="generic" label="Generic Prompt Management">
```yaml
prompts:
- prompt_id: "simple_prompt"
litellm_params:
prompt_integration: "generic_prompt_management"
provider_specific_query_params:
project_name: litellm
slug: hello-world-prompt-2bac
api_base: http://localhost:8080
api_key: os.environ/GENERIC_PROMPT_API_KEY
ignore_prompt_manager_model: true # optional
ignore_prompt_manager_optional_params: true # optional
```
**What you need to implement:**
A GET endpoint at `/beta/litellm_prompt_management` that returns:
```json
{
"prompt_id": "simple_prompt",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant."
},
{
"role": "user",
"content": "Help me with {task}"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 500
}
}
```
**Benefits:**
- No PR required - integrate any prompt management system
- Full control over your prompt storage and versioning
- Support for variable substitution with `{variable}` syntax
- Custom query parameters for filtering and access control
**Learn more:** [Generic Prompt Management API Documentation](../adding_provider/generic_prompt_management_api)
</TabItem>
</Tabs>

8
docs/my-website/docs/proxy/release_cycle.md
View file

@ -22,4 +22,10 @@ Stable releases come out every week (typically Sunday)
- 'patch' bumps: extremely minor addition that doesn't affect any existing functionality or add any user-facing features. (e.g. a 'created_at' column in a database table)
- 'minor' bumps: add a new feature or a new database table that is backward compatible.
- 'major' bumps: break backward compatibility.
- 'major' bumps: break backward compatibility.
### Enterprise Support
- Stable releases come out every week. Once a new one is available, we no longer provide support for an older one.
- If there is a MAJOR change (according to semvar conventions - e.g. 1.x.x -> 2.x.x), we can provide support for upto 90 days on the prior stable image.

4
docs/my-website/docs/proxy/request_headers.md
View file

@ -20,6 +20,10 @@ By default, LiteLLM does not forward client headers to LLM provider APIs. Howeve
`x-litellm-spend-logs-metadata`: Optional[str]: JSON string containing custom metadata to include in spend logs. Example: `{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}`. [Learn More](../proxy/enterprise#tracking-spend-with-custom-metadata)
`x-litellm-customer-id`: Optional[str]: Standard header for passing a customer/end-user ID. Always checked without any configuration. [Learn More](./customers)
`x-litellm-end-user-id`: Optional[str]: Standard header for passing a customer/end-user ID. Always checked without any configuration. [Learn More](./customers)
## Anthropic Headers
`anthropic-version` Optional[str]: The version of the Anthropic API to use.

1
docs/my-website/docs/proxy/team_budgets.md
View file

@ -8,7 +8,6 @@ import TabItem from '@theme/TabItem';
# Pre-Requisites
- You must set up a Postgres database (e.g. Supabase, Neon, etc.)
- To enable team member rate limits, set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` **before starting the proxy server**. Without this, team member rate limits will not be enforced.
## Default Budget for Auto-Generated JWT Teams

11
docs/my-website/docs/proxy/users.md
View file

@ -68,13 +68,6 @@ You can:
**Step-by step tutorial on setting, resetting budgets on Teams here (API or using Admin UI)**
> **Prerequisite:**
> To enable team member rate limits, you must set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` before starting the proxy server. Without this, team member rate limits will not be enforced.
👉 [https://docs.litellm.ai/docs/proxy/team_budgets](https://docs.litellm.ai/docs/proxy/team_budgets)
:::
#### **Add budgets to teams**
```shell
@ -822,12 +815,10 @@ Expected Response:
}
```
### [BETA] Multi-instance rate limiting
### Multi-instance rate limiting
Enable multi-instance rate limiting with the env var `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"`
**Important Notes:**
- Setting `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"` is required for team member rate limits to function, not just for multi-instance scenarios.
- **Rate limits do not apply to proxy admin users.**
- When testing rate limits, use internal user roles (non-admin) to ensure limits are enforced as expected.

7
docs/my-website/docs/proxy/virtual_keys.md
View file

@ -549,11 +549,14 @@ curl 'http://localhost:4000/key/sk-1234/regenerate' \
"models": [
"gpt-4",
"gpt-3.5-turbo"
]
],
"grace_period": "48h"
}'
```
**Grace period (optional)**: Set `grace_period` (e.g. `"24h"`, `"2d"`, `"1w"`) to keep the old key valid for a transitional period. Both old and new keys work until the grace period elapses, enabling seamless cutover without production downtime. Omitted or empty = immediate revoke. Can also be set via `LITELLM_KEY_ROTATION_GRACE_PERIOD` env var for scheduled rotations.
**Read More**
- [Write rotated keys to secrets manager](https://docs.litellm.ai/docs/secret#aws-secret-manager)
@ -640,11 +643,13 @@ Set these environment variables when starting the proxy:
|----------|-------------|---------|
| `LITELLM_KEY_ROTATION_ENABLED` | Enable the rotation worker | `false` |
| `LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS` | How often to scan for keys to rotate (in seconds) | `86400` (24 hours) |
| `LITELLM_KEY_ROTATION_GRACE_PERIOD` | Duration to keep old key valid after rotation (e.g. `24h`, `2d`) | `""` (immediate revoke) |
**Example:**
```bash
export LITELLM_KEY_ROTATION_ENABLED=true
export LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS=3600 # Check every hour
export LITELLM_KEY_ROTATION_GRACE_PERIOD=48h # Keep old key valid for 48h during cutover
litellm --config config.yaml
```

19
docs/my-website/docs/reasoning_content.md
View file

@ -642,6 +642,25 @@ model_list:
model: openai/responses/gpt-5-mini
```
**Per-model configuration** (recommended when using Open WebUI or clients that cannot set `extra_body`):
```yaml
model_list:
- model_name: gpt-5.1
litellm_params:
model: openai/gpt-5.1
# String format - uses reasoning_auto_summary for summary when set
reasoning_effort: "high"
model_info:
mode: responses # if using Responses API bridge
- model_name: gpt-5.1-with-summary
litellm_params:
model: openai/gpt-5.1
# Dict format - explicit control over effort and summary
reasoning_effort: {"effort": "high", "summary": "detailed"}
```
</TabItem>
</Tabs>

47
docs/my-website/docs/rerank.md
View file

@ -8,15 +8,15 @@ LiteLLM Follows the [cohere api request / response for the rerank api](https://c
## Overview
| Feature | Supported | Notes |
|---------|-----------|-------|
| Cost Tracking | ✅ | Works with all supported models |
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input query only (not documents) |
| Supported Providers | Cohere, Together AI, Azure AI, DeepInfra, Nvidia NIM, Infinity, Fireworks AI, Voyage AI | |
| Feature | Supported | Notes |
|---------|-----------------------------------------------------------------------------------------------------|-------|
| Cost Tracking | ✅ | Works with all supported models |
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input query only (not documents) |
| Supported Providers | Cohere, Together AI, Azure AI, DeepInfra, Nvidia NIM, Infinity, Fireworks AI, Voyage AI, watsonx.ai | |
## **LiteLLM Python SDK Usage**
### Quick Start
@ -123,17 +123,18 @@ curl http://0.0.0.0:4000/rerank \
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
| Provider | Link to Usage |
|-------------|--------------------|
| Cohere (v1 + v2 clients) | [Usage](#quick-start) |
| Together AI| [Usage](../docs/providers/togetherai) |
| Azure AI| [Usage](../docs/providers/azure_ai#rerank-endpoint) |
| Jina AI| [Usage](../docs/providers/jina_ai) |
| AWS Bedrock| [Usage](../docs/providers/bedrock#rerank-api) |
| HuggingFace| [Usage](../docs/providers/huggingface_rerank) |
| Infinity| [Usage](../docs/providers/infinity) |
| vLLM| [Usage](../docs/providers/vllm#rerank-endpoint) |
| DeepInfra| [Usage](../docs/providers/deepinfra#rerank-endpoint) |
| Vertex AI| [Usage](../docs/providers/vertex#rerank-api) |
| Fireworks AI| [Usage](../docs/providers/fireworks_ai#rerank-endpoint) |
| Voyage AI| [Usage](../docs/providers/voyage#rerank) |
| Provider | Link to Usage |
|--------------------------|------------------------------------------------------|
| Cohere (v1 + v2 clients) | [Usage](#quick-start) |
| Together AI | [Usage](../docs/providers/togetherai) |
| Azure AI | [Usage](../docs/providers/azure_ai#rerank-endpoint) |
| Jina AI | [Usage](../docs/providers/jina_ai) |
| AWS Bedrock | [Usage](../docs/providers/bedrock#rerank-api) |
| HuggingFace | [Usage](../docs/providers/huggingface_rerank) |
| Infinity | [Usage](../docs/providers/infinity) |
| vLLM | [Usage](../docs/providers/vllm#rerank-endpoint) |
| DeepInfra | [Usage](../docs/providers/deepinfra#rerank-endpoint) |
| Vertex AI | [Usage](../docs/providers/vertex#rerank-api) |
| Fireworks AI | [Usage](../docs/providers/fireworks_ai#rerank-endpoint) |
| Voyage AI | [Usage](../docs/providers/voyage#rerank) |
| IBM watsonx.ai | [Usage](../docs/providers/watsonx/rerank) |

31
docs/my-website/docs/response_api.md
View file

@ -884,7 +884,12 @@ router = litellm.Router(
},
},
],
optional_pre_call_checks=["responses_api_deployment_check"],
# `responses_api_deployment_check` ensures Requests with `previous_response_id`
# are routed to the same deployment. `deployment_affinity` adds sticky sessions
# for requests without `previous_response_id` (useful for implicit caching).
optional_pre_call_checks=["responses_api_deployment_check", "deployment_affinity"],
# Optional (default is 3600 seconds / 1 hour)
deployment_affinity_ttl_seconds=3600,
)
# Initial request
@ -911,7 +916,16 @@ follow_up = await router.aresponses(
#### 1. Setup session continuity on proxy config.yaml
To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks: ["responses_api_deployment_check"]` in your proxy config.yaml.
To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks` in your proxy config.yaml.
- `responses_api_deployment_check`: high priority routing when `previous_response_id` is provided
- `deployment_affinity`: sticky sessions based on user key (applies even without `previous_response_id`)
Notes:
- User-key affinity is keyed on `metadata.user_api_key_hash` (the API key hash). The OpenAI `user` request parameter is an end-user identifier and is intentionally not used for deployment affinity.
- `user_api_key_hash` is already SHA-256, and is used as-is (no double hashing).
- Affinity is scoped by a stable model identifier (the model-map key, e.g. `model_map_information.model_map_key`) so model aliases map to the same stickiness bucket.
- The mapping TTL is controlled by `deployment_affinity_ttl_seconds` (configured on Router init / proxy startup).
```yaml showLineNumbers title="config.yaml with Session Continuity"
model_list:
@ -929,7 +943,11 @@ model_list:
api_base: https://endpoint2.openai.azure.com
router_settings:
optional_pre_call_checks: ["responses_api_deployment_check"]
optional_pre_call_checks:
- responses_api_deployment_check
- deployment_affinity
# Optional (default is 3600 seconds / 1 hour)
deployment_affinity_ttl_seconds: 3600
```
#### 2. Use the OpenAI Python SDK to make requests to LiteLLM Proxy
@ -1029,6 +1047,8 @@ For long-running conversations, you can enable **server-side compaction** so tha
Supported on the OpenAI Responses API when using the `openai` or `azure` provider. Pass `context_management` with a compaction entry and `compact_threshold` (token count; minimum 1000). When the context crosses the threshold, the server compacts in-stream and continues. Chain turns with `previous_response_id` or by appending output items to your next input array. See [OpenAI Compaction guide](https://developers.openai.com/api/docs/guides/compaction) for details.
> **Note:** You can use openai `context_management` format with Anthropic models via LiteLLM via responses API. LiteLLM will automatically translate this format for Anthropic and handle context management for you.
For explicit control over when compaction runs, use the standalone compact endpoint (`POST /v1/responses/compact`) instead.
### Python SDK
@ -1356,8 +1376,3 @@ Response:

1
docs/my-website/docs/search/index.md
View file

@ -276,6 +276,7 @@ The response follows Perplexity's search format with the following structure:
| Firecrawl | `FIRECRAWL_API_KEY` | `firecrawl` |
| SearXNG | `SEARXNG_API_BASE` (required) | `searxng` |
| Linkup | `LINKUP_API_KEY` | `linkup` |
| DuckDuckGo | `DUCKDUCKGO_API_BASE` | `duckduckgo` |
See the individual provider documentation for detailed setup instructions and provider-specific parameters.

90
docs/my-website/docs/troubleshoot/latency_overhead.md Normal file
View file

@ -0,0 +1,90 @@
# Latency Overhead Troubleshooting
Use this guide when you see unexpected latency overhead between LiteLLM proxy and the LLM provider.
## Quick Checklist
1. **Collect the `x-litellm-overhead-duration-ms` response header** — this tells you LiteLLM's total overhead on every request. Start here.
2. **Is DEBUG logging enabled?** This is the #1 cause of latency with large payloads.
3. **Are you sending large base64 payloads?** (images, PDFs) — see [Large Payload Overhead](#large-payload-overhead).
4. **Enable detailed timing headers** to pinpoint where time is spent.
## Diagnostic Headers
### `x-litellm-overhead-duration-ms` (always on)
Every response from LiteLLM includes this header. It shows the total latency overhead in milliseconds added by LiteLLM proxy (i.e. total response time minus the LLM API call time). Collect this on every request to understand your baseline overhead.
```bash
curl -s -D - http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-..." \
-d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}' \
2>&1 | grep x-litellm-overhead-duration-ms
```
### `x-litellm-callback-duration-ms` (always on)
Shows time spent building callback/logging payloads (ms). If this is high (>100ms), your payloads may be too large for efficient logging.
```bash
curl -s -D - http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-..." \
-d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}' \
2>&1 | grep x-litellm
```
### Detailed Timing Breakdown (opt-in)
Set `LITELLM_DETAILED_TIMING=true` to get per-phase timing in response headers:
| Header | What it measures |
|--------|-----------------|
| `x-litellm-timing-pre-processing-ms` | Auth, routing, request processing (before LLM call) |
| `x-litellm-timing-llm-api-ms` | Actual LLM API call duration |
| `x-litellm-timing-post-processing-ms` | Response processing (after LLM returns) |
| `x-litellm-timing-message-copy-ms` | Message copy time in logging layer |
```bash
# Enable detailed timing
export LITELLM_DETAILED_TIMING=true
```
## Large Payload Overhead
When sending large payloads (>1MB, e.g. base64-encoded images/PDFs), three things can add overhead:
### 1. DEBUG Logging (most common)
When `LITELLM_LOG=DEBUG` or `set_verbose=True` is enabled, every request payload is serialized with `json.dumps(indent=4)` synchronously. For a 2MB+ payload, this alone can take **2-5 seconds**.
**Fix:** Don't use DEBUG logging in production. Use `INFO` level instead:
```bash
export LITELLM_LOG=INFO
```
If you need DEBUG logging but have large payloads, you can increase the size threshold for full payload logging:
```bash
# Only fully serialize payloads under 100KB for DEBUG logs (default)
export MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG=102400
```
### 2. Base64 in Logging Payloads
Callback payloads (sent to Langfuse, etc.) include message content. Large base64 strings are automatically truncated to size placeholders in logging payloads.
You can control the truncation threshold:
```bash
# Max base64 characters before truncation (default: 64)
export MAX_BASE64_LENGTH_FOR_LOGGING=64
```
## Environment Variables Reference
| Variable | Default | Description |
|----------|---------|-------------|
| `LITELLM_DETAILED_TIMING` | `false` | Enable per-phase timing headers |
| `MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG` | `102400` | Max payload bytes for full DEBUG serialization |
| `MAX_BASE64_LENGTH_FOR_LOGGING` | `64` | Max base64 chars before truncation in logging |

BIN
docs/my-website/img/passthrough_method_setup.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 77 KiB

BIN
docs/my-website/img/passthrough_query_default.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 84 KiB

BIN
docs/my-website/img/project_spend.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 850 KiB

7
docs/my-website/release_notes/v1.81.12.md
View file

@ -48,6 +48,13 @@ pip install litellm==1.81.12.rc1
- **Responses API `shell` Tool & `context_management` support** - [Server-side context management (compaction) and Shell tool support for the OpenAI Responses API](../../docs/response_api)
- **Access Groups** - [Create access groups to manage model, MCP server, and agent access across teams and keys](../../docs/proxy/access_groups)
- **50+ New Bedrock Regional Model Entries** - DeepSeek V3.2, MiniMax M2.1, Kimi K2.5, Qwen3 Coder Next, and NVIDIA Nemotron Nano across multiple regions
- **Add Semgrep & fix OOMs** - [Static analysis rules and out-of-memory fixes](#add-semgrep--fix-ooms) - [PR #20912](https://github.com/BerriAI/litellm/pull/20912)
---
## Add Semgrep & fix OOMs
This release fixes out-of-memory (OOM) risks from unbounded `asyncio.Queue()` usage. Log queues (e.g. GCS bucket) and DB spend-update queues were previously unbounded and could grow without limit under load. They now use a configurable max size (`LITELLM_ASYNCIO_QUEUE_MAXSIZE`, default 1000); when full, queues flush immediately to make room instead of growing memory. A Semgrep rule (`.semgrep/rules/python/unbounded-memory.yml`) was added to flag similar unbounded-memory patterns in future code. [PR #20912](https://github.com/BerriAI/litellm/pull/20912)
---

31
docs/my-website/sidebars.js
View file

@ -120,6 +120,13 @@ const sidebars = {
type: "category",
label: "[Beta] Prompt Management",
items: [
{
type: "category",
label: "Contributing to Prompt Management",
items: [
"adding_provider/generic_prompt_management_api",
]
},
"proxy/litellm_prompt_management",
"proxy/custom_prompt_management",
"proxy/native_litellm_prompt",
@ -176,6 +183,7 @@ const sidebars = {
"tutorials/copilotkit_sdk",
"tutorials/google_adk",
"tutorials/livekit_xai_realtime",
"projects/openai-agents"
]
},
@ -402,6 +410,7 @@ const sidebars = {
items: [
"proxy/users",
"proxy/team_budgets",
"proxy/project_management",
"proxy/ui_team_soft_budget_alerts",
"proxy/tag_budgets",
"proxy/customers",
@ -572,6 +581,7 @@ const sidebars = {
"proxy/managed_finetuning",
]
},
"evals_api",
"generateContent",
"apply_guardrail",
"bedrock_invoke",
@ -772,13 +782,13 @@ const sidebars = {
"providers/bedrock_batches",
"providers/bedrock_realtime_with_audio",
"providers/aws_polly",
"providers/bedrock_vector_store",
]
},
"providers/litellm_proxy",
"providers/abliteration",
"providers/ai21",
"providers/aiml",
"providers/bedrock_vector_store",
]
},
"providers/litellm_proxy",
"providers/abliteration",
"providers/ai21",
"providers/aiml",
"providers/aleph_alpha",
"providers/amazon_nova",
"providers/anyscale",
@ -935,6 +945,7 @@ const sidebars = {
"providers/anthropic_tool_search",
"guides/code_interpreter",
"completion/message_trimming",
"completion/message_sanitization",
"completion/model_alias",
"completion/mock_requests",
"completion/predict_outputs",
@ -1111,6 +1122,7 @@ const sidebars = {
type: "category",
label: "Performance / Latency",
items: [
"troubleshoot/latency_overhead",
"troubleshoot/cpu_issues",
"troubleshoot/memory_issues",
"troubleshoot/spend_queue_warnings",
@ -1125,6 +1137,11 @@ const sidebars = {
type: "category",
label: "Blog",
items: [
{
type: "link",
label: "Day 0 Support: Claude Sonnet 4.6",
href: "/blog/claude_sonnet_4_6",
},
{
type: "link",
label: "Incident: Broken Model Cost Map",

11
docs/my-website/src/pages/troubleshoot.md
View file

@ -1,11 +0,0 @@
# Troubleshooting
## Stable Version
If you're running into problems with installation / Usage
Use the stable version of litellm
```
pip install litellm==0.1.345
```

BIN
docs/my-website/static/img/project_spend.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 850 KiB

620
enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py
View file

@ -1,309 +1,311 @@
"""
PagerDuty Alerting Integration
Handles two types of alerts:
- High LLM API Failure Rate. Configure X fails in Y seconds to trigger an alert.
- High Number of Hanging LLM Requests. Configure X hangs in Y seconds to trigger an alert.
Note: This is a Free feature on the regular litellm docker image.
However, this is under the enterprise license
"""
import asyncio
import os
from datetime import datetime, timedelta, timezone
from typing import List, Literal, Optional, Union
from litellm._logging import verbose_logger
from litellm.caching import DualCache
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.integrations.pagerduty import (
AlertingConfig,
PagerDutyInternalEvent,
PagerDutyPayload,
PagerDutyRequestBody,
)
from litellm.types.utils import (
CallTypesLiteral,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD = 60
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS = 600
class PagerDutyAlerting(SlackAlerting):
"""
Tracks failed requests and hanging requests separately.
If threshold is crossed for either type, triggers a PagerDuty alert.
"""
def __init__(
self, alerting_args: Optional[Union[AlertingConfig, dict]] = None, **kwargs
):
super().__init__()
_api_key = os.getenv("PAGERDUTY_API_KEY")
if not _api_key:
raise ValueError("PAGERDUTY_API_KEY is not set")
self.api_key: str = _api_key
alerting_args = alerting_args or {}
self.pagerduty_alerting_args: AlertingConfig = AlertingConfig(
failure_threshold=alerting_args.get(
"failure_threshold", PAGERDUTY_DEFAULT_FAILURE_THRESHOLD
),
failure_threshold_window_seconds=alerting_args.get(
"failure_threshold_window_seconds",
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS,
),
hanging_threshold_seconds=alerting_args.get(
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
),
hanging_threshold_window_seconds=alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
),
)
# Separate storage for failures vs. hangs
self._failure_events: List[PagerDutyInternalEvent] = []
self._hanging_events: List[PagerDutyInternalEvent] = []
# ------------------ MAIN LOGIC ------------------ #
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
"""
Record a failure event. Only send an alert to PagerDuty if the
configured *failure* threshold is exceeded in the specified window.
"""
now = datetime.now(timezone.utc)
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object"
)
if not standard_logging_payload:
raise ValueError(
"standard_logging_object is required for PagerDutyAlerting"
)
# Extract error details
error_info: Optional[StandardLoggingPayloadErrorInformation] = (
standard_logging_payload.get("error_information") or {}
)
_meta = standard_logging_payload.get("metadata") or {}
self._failure_events.append(
PagerDutyInternalEvent(
failure_event_type="failed_response",
timestamp=now,
error_class=error_info.get("error_class"),
error_code=error_info.get("error_code"),
error_llm_provider=error_info.get("llm_provider"),
user_api_key_hash=_meta.get("user_api_key_hash"),
user_api_key_alias=_meta.get("user_api_key_alias"),
user_api_key_spend=_meta.get("user_api_key_spend"),
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
user_api_key_org_id=_meta.get("user_api_key_org_id"),
user_api_key_team_id=_meta.get("user_api_key_team_id"),
user_api_key_user_id=_meta.get("user_api_key_user_id"),
user_api_key_team_alias=_meta.get("user_api_key_team_alias"),
user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"),
user_api_key_user_email=_meta.get("user_api_key_user_email"),
user_api_key_request_route=_meta.get("user_api_key_request_route"),
user_api_key_auth_metadata=_meta.get("user_api_key_auth_metadata"),
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get(
"failure_threshold_window_seconds", 60
)
threshold = self.pagerduty_alerting_args.get("failure_threshold", 1)
# If threshold is crossed, send PD alert for failures
await self._send_alert_if_thresholds_crossed(
events=self._failure_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High LLM API Failure Rate",
)
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: CallTypesLiteral,
) -> Optional[Union[Exception, str, dict]]:
"""
Example of detecting hanging requests by waiting a given threshold.
If the request didn't finish by then, we treat it as 'hanging'.
"""
verbose_logger.info("Inside Proxy Logging Pre-call hook!")
asyncio.create_task(
self.hanging_response_handler(
request_data=data, user_api_key_dict=user_api_key_dict
)
)
return None
async def hanging_response_handler(
self, request_data: Optional[dict], user_api_key_dict: UserAPIKeyAuth
):
"""
Checks if request completed by the time 'hanging_threshold_seconds' elapses.
If not, we classify it as a hanging request.
"""
verbose_logger.debug(
f"Inside Hanging Response Handler!..sleeping for {self.pagerduty_alerting_args.get('hanging_threshold_seconds', PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)} seconds"
)
await asyncio.sleep(
self.pagerduty_alerting_args.get(
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
)
)
if await self._request_is_completed(request_data=request_data):
return # It's not hanging if completed
# Otherwise, record it as hanging
self._hanging_events.append(
PagerDutyInternalEvent(
failure_event_type="hanging_response",
timestamp=datetime.now(timezone.utc),
error_class="HangingRequest",
error_code="HangingRequest",
error_llm_provider="HangingRequest",
user_api_key_hash=user_api_key_dict.api_key,
user_api_key_alias=user_api_key_dict.key_alias,
user_api_key_spend=user_api_key_dict.spend,
user_api_key_max_budget=user_api_key_dict.max_budget,
user_api_key_budget_reset_at=(
user_api_key_dict.budget_reset_at.isoformat()
if user_api_key_dict.budget_reset_at
else None
),
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_team_id=user_api_key_dict.team_id,
user_api_key_user_id=user_api_key_dict.user_id,
user_api_key_team_alias=user_api_key_dict.team_alias,
user_api_key_end_user_id=user_api_key_dict.end_user_id,
user_api_key_user_email=user_api_key_dict.user_email,
user_api_key_request_route=user_api_key_dict.request_route,
user_api_key_auth_metadata=user_api_key_dict.metadata,
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
)
threshold: int = self.pagerduty_alerting_args.get(
"hanging_threshold_fails", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
)
# If threshold is crossed, send PD alert for hangs
await self._send_alert_if_thresholds_crossed(
events=self._hanging_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High Number of Hanging LLM Requests",
)
# ------------------ HELPERS ------------------ #
async def _send_alert_if_thresholds_crossed(
self,
events: List[PagerDutyInternalEvent],
window_seconds: int,
threshold: int,
alert_prefix: str,
):
"""
1. Prune old events
2. If threshold is reached, build alert, send to PagerDuty
3. Clear those events
"""
cutoff = datetime.now(timezone.utc) - timedelta(seconds=window_seconds)
pruned = [e for e in events if e.get("timestamp", datetime.min) > cutoff]
# Update the reference list
events.clear()
events.extend(pruned)
# Check threshold
verbose_logger.debug(
f"Have {len(events)} events in the last {window_seconds} seconds. Threshold is {threshold}"
)
if len(events) >= threshold:
# Build short summary of last N events
error_summaries = self._build_error_summaries(events, max_errors=5)
alert_message = (
f"{alert_prefix}: {len(events)} in the last {window_seconds} seconds."
)
custom_details = {"recent_errors": error_summaries}
await self.send_alert_to_pagerduty(
alert_message=alert_message,
custom_details=custom_details,
)
# Clear them after sending an alert, so we don't spam
events.clear()
def _build_error_summaries(
self, events: List[PagerDutyInternalEvent], max_errors: int = 5
) -> List[PagerDutyInternalEvent]:
"""
Build short text summaries for the last `max_errors`.
Example: "ValueError (code: 500, provider: openai)"
"""
recent = events[-max_errors:]
summaries = []
for fe in recent:
# If any of these is None, show "N/A" to avoid messing up the summary string
fe.pop("timestamp")
summaries.append(fe)
return summaries
async def send_alert_to_pagerduty(self, alert_message: str, custom_details: dict):
"""
Send [critical] Alert to PagerDuty
https://developer.pagerduty.com/api-reference/YXBpOjI3NDgyNjU-pager-duty-v2-events-api
"""
try:
verbose_logger.debug(f"Sending alert to PagerDuty: {alert_message}")
async_client: AsyncHTTPHandler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
payload: PagerDutyRequestBody = PagerDutyRequestBody(
payload=PagerDutyPayload(
summary=alert_message,
severity="critical",
source="LiteLLM Alert",
component="LiteLLM",
custom_details=custom_details,
),
routing_key=self.api_key,
event_action="trigger",
)
return await async_client.post(
url="https://events.pagerduty.com/v2/enqueue",
json=dict(payload),
headers={"Content-Type": "application/json"},
)
except Exception as e:
verbose_logger.exception(f"Error sending alert to PagerDuty: {e}")
"""
PagerDuty Alerting Integration
Handles two types of alerts:
- High LLM API Failure Rate. Configure X fails in Y seconds to trigger an alert.
- High Number of Hanging LLM Requests. Configure X hangs in Y seconds to trigger an alert.
Note: This is a Free feature on the regular litellm docker image.
However, this is under the enterprise license
"""
import asyncio
import os
from datetime import datetime, timedelta, timezone
from typing import List, Optional, Union
from litellm._logging import verbose_logger
from litellm.caching import DualCache
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.integrations.pagerduty import (
AlertingConfig,
PagerDutyInternalEvent,
PagerDutyPayload,
PagerDutyRequestBody,
)
from litellm.types.utils import (
CallTypesLiteral,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD = 60
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS = 60
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS = 600
class PagerDutyAlerting(SlackAlerting):
"""
Tracks failed requests and hanging requests separately.
If threshold is crossed for either type, triggers a PagerDuty alert.
"""
def __init__(
self, alerting_args: Optional[Union[AlertingConfig, dict]] = None, **kwargs
):
super().__init__()
_api_key = os.getenv("PAGERDUTY_API_KEY")
if not _api_key:
raise ValueError("PAGERDUTY_API_KEY is not set")
self.api_key: str = _api_key
alerting_args = alerting_args or {}
self.pagerduty_alerting_args: AlertingConfig = AlertingConfig(
failure_threshold=alerting_args.get(
"failure_threshold", PAGERDUTY_DEFAULT_FAILURE_THRESHOLD
),
failure_threshold_window_seconds=alerting_args.get(
"failure_threshold_window_seconds",
PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS,
),
hanging_threshold_seconds=alerting_args.get(
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
),
hanging_threshold_window_seconds=alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
),
)
# Separate storage for failures vs. hangs
self._failure_events: List[PagerDutyInternalEvent] = []
self._hanging_events: List[PagerDutyInternalEvent] = []
# ------------------ MAIN LOGIC ------------------ #
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
"""
Record a failure event. Only send an alert to PagerDuty if the
configured *failure* threshold is exceeded in the specified window.
"""
now = datetime.now(timezone.utc)
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object"
)
if not standard_logging_payload:
raise ValueError(
"standard_logging_object is required for PagerDutyAlerting"
)
# Extract error details
error_info: Optional[StandardLoggingPayloadErrorInformation] = (
standard_logging_payload.get("error_information") or {}
)
_meta = standard_logging_payload.get("metadata") or {}
self._failure_events.append(
PagerDutyInternalEvent(
failure_event_type="failed_response",
timestamp=now,
error_class=error_info.get("error_class"),
error_code=error_info.get("error_code"),
error_llm_provider=error_info.get("llm_provider"),
user_api_key_hash=_meta.get("user_api_key_hash"),
user_api_key_alias=_meta.get("user_api_key_alias"),
user_api_key_spend=_meta.get("user_api_key_spend"),
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
user_api_key_org_id=_meta.get("user_api_key_org_id"),
user_api_key_team_id=_meta.get("user_api_key_team_id"),
user_api_key_project_id=_meta.get("user_api_key_project_id"),
user_api_key_user_id=_meta.get("user_api_key_user_id"),
user_api_key_team_alias=_meta.get("user_api_key_team_alias"),
user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"),
user_api_key_user_email=_meta.get("user_api_key_user_email"),
user_api_key_request_route=_meta.get("user_api_key_request_route"),
user_api_key_auth_metadata=_meta.get("user_api_key_auth_metadata"),
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get(
"failure_threshold_window_seconds", 60
)
threshold = self.pagerduty_alerting_args.get("failure_threshold", 1)
# If threshold is crossed, send PD alert for failures
await self._send_alert_if_thresholds_crossed(
events=self._failure_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High LLM API Failure Rate",
)
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: CallTypesLiteral,
) -> Optional[Union[Exception, str, dict]]:
"""
Example of detecting hanging requests by waiting a given threshold.
If the request didn't finish by then, we treat it as 'hanging'.
"""
verbose_logger.info("Inside Proxy Logging Pre-call hook!")
asyncio.create_task(
self.hanging_response_handler(
request_data=data, user_api_key_dict=user_api_key_dict
)
)
return None
async def hanging_response_handler(
self, request_data: Optional[dict], user_api_key_dict: UserAPIKeyAuth
):
"""
Checks if request completed by the time 'hanging_threshold_seconds' elapses.
If not, we classify it as a hanging request.
"""
verbose_logger.debug(
f"Inside Hanging Response Handler!..sleeping for {self.pagerduty_alerting_args.get('hanging_threshold_seconds', PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS)} seconds"
)
await asyncio.sleep(
self.pagerduty_alerting_args.get(
"hanging_threshold_seconds", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
)
)
if await self._request_is_completed(request_data=request_data):
return # It's not hanging if completed
# Otherwise, record it as hanging
self._hanging_events.append(
PagerDutyInternalEvent(
failure_event_type="hanging_response",
timestamp=datetime.now(timezone.utc),
error_class="HangingRequest",
error_code="HangingRequest",
error_llm_provider="HangingRequest",
user_api_key_hash=user_api_key_dict.api_key,
user_api_key_alias=user_api_key_dict.key_alias,
user_api_key_spend=user_api_key_dict.spend,
user_api_key_max_budget=user_api_key_dict.max_budget,
user_api_key_budget_reset_at=(
user_api_key_dict.budget_reset_at.isoformat()
if user_api_key_dict.budget_reset_at
else None
),
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_team_id=user_api_key_dict.team_id,
user_api_key_project_id=user_api_key_dict.project_id,
user_api_key_user_id=user_api_key_dict.user_id,
user_api_key_team_alias=user_api_key_dict.team_alias,
user_api_key_end_user_id=user_api_key_dict.end_user_id,
user_api_key_user_email=user_api_key_dict.user_email,
user_api_key_request_route=user_api_key_dict.request_route,
user_api_key_auth_metadata=user_api_key_dict.metadata,
)
)
# Prune + Possibly alert
window_seconds = self.pagerduty_alerting_args.get(
"hanging_threshold_window_seconds",
PAGERDUTY_DEFAULT_HANGING_THRESHOLD_WINDOW_SECONDS,
)
threshold: int = self.pagerduty_alerting_args.get(
"hanging_threshold_fails", PAGERDUTY_DEFAULT_HANGING_THRESHOLD_SECONDS
)
# If threshold is crossed, send PD alert for hangs
await self._send_alert_if_thresholds_crossed(
events=self._hanging_events,
window_seconds=window_seconds,
threshold=threshold,
alert_prefix="High Number of Hanging LLM Requests",
)
# ------------------ HELPERS ------------------ #
async def _send_alert_if_thresholds_crossed(
self,
events: List[PagerDutyInternalEvent],
window_seconds: int,
threshold: int,
alert_prefix: str,
):
"""
1. Prune old events
2. If threshold is reached, build alert, send to PagerDuty
3. Clear those events
"""
cutoff = datetime.now(timezone.utc) - timedelta(seconds=window_seconds)
pruned = [e for e in events if e.get("timestamp", datetime.min) > cutoff]
# Update the reference list
events.clear()
events.extend(pruned)
# Check threshold
verbose_logger.debug(
f"Have {len(events)} events in the last {window_seconds} seconds. Threshold is {threshold}"
)
if len(events) >= threshold:
# Build short summary of last N events
error_summaries = self._build_error_summaries(events, max_errors=5)
alert_message = (
f"{alert_prefix}: {len(events)} in the last {window_seconds} seconds."
)
custom_details = {"recent_errors": error_summaries}
await self.send_alert_to_pagerduty(
alert_message=alert_message,
custom_details=custom_details,
)
# Clear them after sending an alert, so we don't spam
events.clear()
def _build_error_summaries(
self, events: List[PagerDutyInternalEvent], max_errors: int = 5
) -> List[PagerDutyInternalEvent]:
"""
Build short text summaries for the last `max_errors`.
Example: "ValueError (code: 500, provider: openai)"
"""
recent = events[-max_errors:]
summaries = []
for fe in recent:
# If any of these is None, show "N/A" to avoid messing up the summary string
fe.pop("timestamp")
summaries.append(fe)
return summaries
async def send_alert_to_pagerduty(self, alert_message: str, custom_details: dict):
"""
Send [critical] Alert to PagerDuty
https://developer.pagerduty.com/api-reference/YXBpOjI3NDgyNjU-pager-duty-v2-events-api
"""
try:
verbose_logger.debug(f"Sending alert to PagerDuty: {alert_message}")
async_client: AsyncHTTPHandler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
payload: PagerDutyRequestBody = PagerDutyRequestBody(
payload=PagerDutyPayload(
summary=alert_message,
severity="critical",
source="LiteLLM Alert",
component="LiteLLM",
custom_details=custom_details,
),
routing_key=self.api_key,
event_action="trigger",
)
return await async_client.post(
url="https://events.pagerduty.com/v2/enqueue",
json=dict(payload),
headers={"Content-Type": "application/json"},
)
except Exception as e:
verbose_logger.exception(f"Error sending alert to PagerDuty: {e}")

51
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -4,7 +4,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
from litellm._uuid import uuid
from datetime import datetime
from typing import TYPE_CHECKING, Optional, cast
from typing import TYPE_CHECKING, Optional
from litellm._logging import verbose_proxy_logger
@ -35,14 +35,11 @@ class CheckBatchCost:
- if not, return False
- if so, return True
"""
from litellm_enterprise.proxy.hooks.managed_files import (
_PROXY_LiteLLMManagedFiles,
)
from litellm.batches.batch_utils import (
_get_file_content_as_dictionary,
calculate_batch_cost_and_usage,
)
from litellm.files.main import afile_content
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.proxy.openai_files_endpoints.common_utils import (
@ -102,31 +99,41 @@ class CheckBatchCost:
continue
## RETRIEVE THE BATCH JOB OUTPUT FILE
managed_files_obj = cast(
Optional[_PROXY_LiteLLMManagedFiles],
self.proxy_logging_obj.get_proxy_hook("managed_files"),
)
if (
response.status == "completed"
and response.output_file_id is not None
and managed_files_obj is not None
):
verbose_proxy_logger.info(
f"Batch ID: {batch_id} is complete, tracking cost and usage"
)
# track cost
model_file_id_mapping = {
response.output_file_id: {model_id: response.output_file_id}
}
_file_content = await managed_files_obj.afile_content(
file_id=response.output_file_id,
litellm_parent_otel_span=None,
llm_router=self.llm_router,
model_file_id_mapping=model_file_id_mapping,
# This background job runs as default_user_id, so going through the HTTP endpoint
# would trigger check_managed_file_id_access and get 403. Instead, extract the raw
# provider file ID and call afile_content directly with deployment credentials.
raw_output_file_id = response.output_file_id
decoded = _is_base64_encoded_unified_file_id(raw_output_file_id)
if decoded:
try:
raw_output_file_id = decoded.split("llm_output_file_id,")[1].split(";")[0]
except (IndexError, AttributeError):
pass
credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
_file_content = await afile_content(
file_id=raw_output_file_id,
**credentials,
)
# Access content - handle both direct attribute and method call
if hasattr(_file_content, 'content'):
content_bytes = _file_content.content
elif hasattr(_file_content, 'read'):
content_bytes = await _file_content.read()
else:
content_bytes = _file_content
file_content_as_dict = _get_file_content_as_dictionary(
_file_content.content
content_bytes
)
deployment_info = self.llm_router.get_deployment(model_id=model_id)
@ -143,11 +150,15 @@ class CheckBatchCost:
custom_llm_provider=custom_llm_provider,
)
# Pass deployment model_info so custom batch pricing
# (input_cost_per_token_batches etc.) is used for cost calc
deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}
batch_cost, batch_usage, batch_models = (
await calculate_batch_cost_and_usage(
file_content_dictionary=file_content_as_dict,
custom_llm_provider=llm_provider, # type: ignore
model_name=model_name,
model_info=deployment_model_info,
)
)
logging_obj = LiteLLMLogging(

205
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -230,12 +230,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if managed_file:
return managed_file.created_by == user_id
return False
raise HTTPException(
status_code=404,
detail=f"File not found: {unified_file_id}",
)
async def can_user_call_unified_object_id(
self, unified_object_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
## check if the user has access to the unified object id
## check if the user has access to the unified object id
user_id = user_api_key_dict.user_id
managed_object = (
@ -246,7 +248,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if managed_object:
return managed_object.created_by == user_id
return True # don't raise error if managed object is not found
raise HTTPException(
status_code=404,
detail=f"Object not found: {unified_object_id}",
)
async def list_user_batches(
self,
@ -911,15 +916,24 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
setattr(response, file_attr, unified_file_id)
# Fetch the actual file object from the provider
# Use llm_router credentials when available. Without credentials,
# Azure and other auth-required providers return 500/401.
file_object = None
try:
# Use litellm to retrieve the file object from the provider
from litellm import afile_retrieve
file_object = await afile_retrieve(
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
file_id=original_file_id
)
# Import module and use getattr for better testability with mocks
import litellm.proxy.proxy_server as proxy_server_module
_llm_router = getattr(proxy_server_module, 'llm_router', None)
if _llm_router is not None and model_id:
_creds = _llm_router.get_deployment_credentials_with_provider(model_id) or {}
file_object = await litellm.afile_retrieve(
file_id=original_file_id,
**_creds,
)
else:
file_object = await litellm.afile_retrieve(
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
file_id=original_file_id,
)
verbose_logger.debug(
f"Successfully retrieved file object for {file_attr}={original_file_id}"
)
@ -1004,8 +1018,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
raise Exception(f"LiteLLM Managed File object with id={file_id} not found")
# Case 2: Managed file and the file object exists in the database
# The stored file_object has the raw provider ID. Replace with the unified ID
# so callers see a consistent ID (matching Case 3 which does response.id = file_id).
if stored_file_object and stored_file_object.file_object:
return stored_file_object.file_object
# Use model_copy to ensure the ID update persists (Pydantic v2 compatibility)
response = stored_file_object.file_object.model_copy(update={"id": file_id})
return response
# Case 3: Managed file exists in the database but not the file object (for. e.g the batch task might not have run)
# So we fetch the file object from the provider. We deliberately do not store the result to avoid interfering with batch cost tracking code.
@ -1033,6 +1051,168 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"""Handled in files_endpoints.py"""
return []
def _is_batch_polling_enabled(self) -> bool:
"""
Check if batch cost tracking is actually enabled and running.
Returns:
bool: True if batch cost tracking is active, False otherwise
"""
try:
# Import here to avoid circular dependencies
import litellm.proxy.proxy_server as proxy_server_module
# Check if the scheduler has the batch cost checking job registered
scheduler = getattr(proxy_server_module, 'scheduler', None)
if scheduler is None:
return False
# Check if the check_batch_cost_job exists in the scheduler
try:
job = scheduler.get_job('check_batch_cost_job')
if job is not None:
return True
except Exception:
# Job not found or scheduler doesn't support get_job
pass
return False
except Exception as e:
verbose_logger.warning(
f"Error checking batch polling configuration: {e}. Assuming disabled."
)
return False
async def _get_batches_referencing_file(
self, file_id: str
) -> List[Dict[str, Any]]:
"""
Find batches in non-terminal states that reference this file.
Non-terminal states: validating, in_progress, finalizing
Terminal states: completed, complete, failed, expired, cancelled
Args:
file_id: The unified file ID to check
Returns:
List of batch objects referencing this file in non-terminal state
(max 10 for error message display)
"""
# Prepare list of file IDs to check (both unified and provider IDs)
file_ids_to_check = [file_id]
# Get model-specific file IDs for this unified file ID if it's a managed file
try:
model_file_id_mapping = await self.get_model_file_id_mapping(
[file_id], litellm_parent_otel_span=None
)
if model_file_id_mapping and file_id in model_file_id_mapping:
# Add all provider file IDs for this unified file
provider_file_ids = list(model_file_id_mapping[file_id].values())
file_ids_to_check.extend(provider_file_ids)
except Exception as e:
verbose_logger.debug(
f"Could not get model file ID mapping for {file_id}: {e}. "
f"Will only check unified file ID."
)
MAX_MATCHES_TO_RETURN = 10
batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
where={
"file_purpose": "batch",
"status": {"in": ["validating", "in_progress", "finalizing"]},
},
take=MAX_MATCHES_TO_RETURN,
order={"created_at": "desc"},
)
referencing_batches = []
for batch in batches:
try:
# Parse the batch file_object to check for file references
batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
# Extract file IDs from batch
# Batches typically reference the unified file ID in input_file_id
# Output and error files are generated by the provider
input_file_id = batch_data.get("input_file_id")
output_file_id = batch_data.get("output_file_id")
error_file_id = batch_data.get("error_file_id")
referenced_file_ids = [fid for fid in [input_file_id, output_file_id, error_file_id] if fid]
# Check if any referenced file ID matches the file we're trying to delete
if any(ref_id in file_ids_to_check for ref_id in referenced_file_ids):
referencing_batches.append({
"batch_id": batch.unified_object_id,
"status": batch.status,
"created_at": batch.created_at,
})
except Exception as e:
verbose_logger.warning(
f"Error parsing batch object {batch.unified_object_id}: {e}"
)
continue
return referencing_batches
async def _check_file_deletion_allowed(self, file_id: str) -> None:
"""
Check if file deletion should be blocked due to batch references.
Blocks deletion if:
1. File is referenced by any batch in non-terminal state, AND
2. Batch polling is configured (user wants cost tracking)
Args:
file_id: The unified file ID to check
Raises:
HTTPException: If file deletion should be blocked
"""
# Check if batch polling is enabled
if not self._is_batch_polling_enabled():
# Batch polling not configured, allow deletion
return
# Check if file is referenced by any non-terminal batches
referencing_batches = await self._get_batches_referencing_file(file_id)
if referencing_batches:
# File is referenced by non-terminal batches and polling is enabled
MAX_BATCHES_IN_ERROR = 5 # Limit batches shown in error message for readability
# Show up to MAX_BATCHES_IN_ERROR in the error message
batches_to_show = referencing_batches[:MAX_BATCHES_IN_ERROR]
batch_statuses = [f"{b['batch_id']}: {b['status']}" for b in batches_to_show]
# Determine the count message
count_message = f"{len(referencing_batches)}"
if len(referencing_batches) >= 10: # MAX_MATCHES_TO_RETURN from _get_batches_referencing_file
count_message = "10+"
error_message = (
f"Cannot delete file {file_id}. "
f"The file is referenced by {count_message} batch(es) in non-terminal state"
)
# Add specific batch details if not too many
if len(referencing_batches) <= MAX_BATCHES_IN_ERROR:
error_message += f": {', '.join(batch_statuses)}. "
else:
error_message += f" (showing {MAX_BATCHES_IN_ERROR} most recent): {', '.join(batch_statuses)}. "
error_message += (
f"To delete this file before complete cost tracking, please delete or cancel the referencing batch(es) first. "
f"Alternatively, wait for all batches to complete processing."
)
raise HTTPException(
status_code=400,
detail=error_message,
)
async def afile_delete(
self,
file_id: str,
@ -1041,6 +1221,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
**data: Dict,
) -> OpenAIFileObject:
# Check if file deletion should be blocked due to batch references
await self._check_file_deletion_allowed(file_id)
# file_id = convert_b64_uid_to_unified_uid(file_id)
model_file_id_mapping = await self.get_model_file_id_mapping(
[file_id], litellm_parent_otel_span

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.40-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.40.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.41-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.41.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.42-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.42.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.43-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.43.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.44-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.44.tar.gz vendored Normal file
View file

Binary file not shown.

35
litellm-proxy-extras/litellm_proxy_extras/migrations/20251113000000_add_project_table/migration.sql Normal file
View file

@ -0,0 +1,35 @@
-- CreateTable
CREATE TABLE "LiteLLM_ProjectTable" (
"project_id" TEXT NOT NULL,
"project_alias" TEXT,
"team_id" TEXT,
"budget_id" TEXT,
"metadata" JSONB NOT NULL DEFAULT '{}',
"models" TEXT[],
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
"model_spend" JSONB NOT NULL DEFAULT '{}',
"blocked" BOOLEAN NOT NULL DEFAULT false,
"object_permission_id" TEXT,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT NOT NULL,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT NOT NULL,
CONSTRAINT "LiteLLM_ProjectTable_pkey" PRIMARY KEY ("project_id")
);
-- AddForeignKey
ALTER TABLE "LiteLLM_ProjectTable" ADD CONSTRAINT "LiteLLM_ProjectTable_team_id_fkey" FOREIGN KEY ("team_id") REFERENCES "LiteLLM_TeamTable"("team_id") ON DELETE SET NULL ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "LiteLLM_ProjectTable" ADD CONSTRAINT "LiteLLM_ProjectTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "LiteLLM_ProjectTable" ADD CONSTRAINT "LiteLLM_ProjectTable_object_permission_id_fkey" FOREIGN KEY ("object_permission_id") REFERENCES "LiteLLM_ObjectPermissionTable"("object_permission_id") ON DELETE SET NULL ON UPDATE CASCADE;
-- AlterTable: Add project_id to LiteLLM_VerificationToken
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "project_id" TEXT;
-- AddForeignKey
ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "LiteLLM_VerificationToken_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "LiteLLM_ProjectTable"("project_id") ON DELETE SET NULL ON UPDATE CASCADE;

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20251113000001_add_project_fields/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable: Add new fields to LiteLLM_ProjectTable
ALTER TABLE "LiteLLM_ProjectTable" ADD COLUMN "description" TEXT;
ALTER TABLE "LiteLLM_ProjectTable" ADD COLUMN "model_rpm_limit" JSONB NOT NULL DEFAULT '{}';
ALTER TABLE "LiteLLM_ProjectTable" ADD COLUMN "model_tpm_limit" JSONB NOT NULL DEFAULT '{}';

11
litellm-proxy-extras/litellm_proxy_extras/migrations/20260131150814_add_team_user_to_vector_stores/migration.sql
View file

@ -1,10 +1,13 @@
-- AlterTable
ALTER TABLE "LiteLLM_ManagedVectorStoresTable" ADD COLUMN "team_id" TEXT,
ADD COLUMN "user_id" TEXT;
ALTER TABLE "LiteLLM_ManagedVectorStoresTable"
ADD COLUMN IF NOT EXISTS "team_id" TEXT,
ADD COLUMN IF NOT EXISTS "user_id" TEXT;
-- CreateIndex
CREATE INDEX "LiteLLM_ManagedVectorStoresTable_team_id_idx" ON "LiteLLM_ManagedVectorStoresTable"("team_id");
CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedVectorStoresTable_team_id_idx"
ON "LiteLLM_ManagedVectorStoresTable"("team_id");
-- CreateIndex
CREATE INDEX "LiteLLM_ManagedVectorStoresTable_user_id_idx" ON "LiteLLM_ManagedVectorStoresTable"("user_id");
CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedVectorStoresTable_user_id_idx"
ON "LiteLLM_ManagedVectorStoresTable"("user_id");

19
litellm-proxy-extras/litellm_proxy_extras/migrations/20260203120000_add_deprecated_verification_token_table/migration.sql Normal file
View file

@ -0,0 +1,19 @@
-- CreateTable
CREATE TABLE "LiteLLM_DeprecatedVerificationToken" (
"id" TEXT NOT NULL,
"token" TEXT NOT NULL,
"active_token_id" TEXT NOT NULL,
"revoke_at" TIMESTAMP(3) NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "LiteLLM_DeprecatedVerificationToken_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_DeprecatedVerificationToken_token_key" ON "LiteLLM_DeprecatedVerificationToken"("token");
-- CreateIndex
CREATE INDEX "LiteLLM_DeprecatedVerificationToken_token_revoke_at_idx" ON "LiteLLM_DeprecatedVerificationToken"("token", "revoke_at");
-- CreateIndex
CREATE INDEX "LiteLLM_DeprecatedVerificationToken_revoke_at_idx" ON "LiteLLM_DeprecatedVerificationToken"("revoke_at");

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260214124140_baseline_diff/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- This is an empty migration.

6
litellm-proxy-extras/litellm_proxy_extras/migrations/20260214185341_object_permissions_for_end_users/migration.sql Normal file
View file

@ -0,0 +1,6 @@
-- AlterTable
ALTER TABLE "LiteLLM_EndUserTable" ADD COLUMN "object_permission_id" TEXT;
-- AddForeignKey
ALTER TABLE "LiteLLM_EndUserTable" ADD CONSTRAINT "LiteLLM_EndUserTable_object_permission_id_fkey" FOREIGN KEY ("object_permission_id") REFERENCES "LiteLLM_ObjectPermissionTable"("object_permission_id") ON DELETE SET NULL ON UPDATE CASCADE;

6
litellm-proxy-extras/litellm_proxy_extras/migrations/20260218231534_add_last_active_to_key_table/migration.sql Normal file
View file

@ -0,0 +1,6 @@
-- AlterTable
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN "last_active" TIMESTAMP(3);
-- AlterTable
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "last_active" TIMESTAMP(3);

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260219105005_add_project_id_to_deleted_keys/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN "project_id" TEXT;

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260220124742_add_spec_path_to_mcp_servers/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "spec_path" TEXT;

100
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -24,6 +24,7 @@ model LiteLLM_BudgetTable {
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
organization LiteLLM_OrganizationTable[] // multiple orgs can have the same budget
projects LiteLLM_ProjectTable[] // multiple projects can have the same budget
keys LiteLLM_VerificationToken[] // multiple keys can have the same budget
end_users LiteLLM_EndUserTable[] // multiple end-users can have the same budget
tags LiteLLM_TagTable[] // multiple tags can have the same budget
@ -135,6 +136,81 @@ model LiteLLM_TeamTable {
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
litellm_model_table LiteLLM_ModelTable? @relation(fields: [model_id], references: [id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
projects LiteLLM_ProjectTable[]
}
// Projects sit between teams and keys for use-case management
model LiteLLM_ProjectTable {
project_id String @id @default(uuid())
project_alias String?
description String?
team_id String?
budget_id String?
metadata Json @default("{}")
models String[]
spend Float @default(0.0)
model_spend Json @default("{}")
model_rpm_limit Json @default("{}")
model_tpm_limit Json @default("{}")
blocked Boolean @default(false)
object_permission_id String?
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
// Relations
litellm_team_table LiteLLM_TeamTable? @relation(fields: [team_id], references: [team_id])
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
keys LiteLLM_VerificationToken[]
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
}
// Audit table for deleted teams - preserves spend and team information for historical tracking
model LiteLLM_DeletedTeamTable {
id String @id @default(uuid())
team_id String // Original team_id
team_alias String?
organization_id String?
object_permission_id String?
admins String[]
members String[]
members_with_roles Json @default("{}")
metadata Json @default("{}")
max_budget Float?
soft_budget Float?
spend Float @default(0.0)
models String[]
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
model_spend Json @default("{}")
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
team_member_permissions String[] @default([])
access_group_ids String[] @default([])
policies String[] @default([])
model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases
allow_team_guardrail_config Boolean @default(false)
// Original timestamps from team creation/updates
created_at DateTime? @map("created_at")
updated_at DateTime? @map("updated_at")
// Deletion metadata
deleted_at DateTime @default(now()) @map("deleted_at")
deleted_by String? @map("deleted_by") // User who deleted the team
deleted_by_api_key String? @map("deleted_by_api_key") // API key hash that performed the deletion
litellm_changed_by String? @map("litellm_changed_by") // From litellm-changed-by header if provided
@@index([team_id])
@@index([deleted_at])
@@index([organization_id])
@@index([team_alias])
@@index([created_at])
}
// Audit table for deleted teams - preserves spend and team information for historical tracking
@ -230,9 +306,11 @@ model LiteLLM_ObjectPermissionTable {
agents String[] @default([])
agent_access_groups String[] @default([])
teams LiteLLM_TeamTable[]
projects LiteLLM_ProjectTable[]
verification_tokens LiteLLM_VerificationToken[]
organizations LiteLLM_OrganizationTable[]
users LiteLLM_UserTable[]
end_users LiteLLM_EndUserTable[]
}
// Holds the MCP server configuration
@ -283,6 +361,7 @@ model LiteLLM_VerificationToken {
router_settings Json? @default("{}")
user_id String?
team_id String?
project_id String?
permissions Json @default("{}")
max_parallel_requests Int?
metadata Json @default("{}")
@ -305,6 +384,7 @@ model LiteLLM_VerificationToken {
created_by String?
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
updated_by String?
last_active DateTime? // When this key was last used
rotation_count Int? @default(0) // Number of times key has been rotated
auto_rotate Boolean? @default(false) // Whether this key should be auto-rotated
rotation_interval String? // How often to rotate (e.g., "30d", "90d")
@ -312,6 +392,7 @@ model LiteLLM_VerificationToken {
key_rotation_at DateTime? // When this key should next be rotated
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
litellm_project_table LiteLLM_ProjectTable? @relation(fields: [project_id], references: [project_id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
// SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub"
@ -325,6 +406,19 @@ model LiteLLM_VerificationToken {
@@index([budget_reset_at, expires])
}
// Deprecated keys during grace period - allows old key to work until revoke_at
model LiteLLM_DeprecatedVerificationToken {
id String @id @default(uuid())
token String // Hashed old key
active_token_id String // Current token hash in LiteLLM_VerificationToken
revoke_at DateTime // When the old key stops working
created_at DateTime @default(now()) @map("created_at")
@@unique([token])
@@index([token, revoke_at])
@@index([revoke_at])
}
// Audit table for deleted keys - preserves spend and key information for historical tracking
model LiteLLM_DeletedVerificationToken {
id String @id @default(uuid())
@ -339,6 +433,7 @@ model LiteLLM_DeletedVerificationToken {
config Json @default("{}")
user_id String?
team_id String?
project_id String?
permissions Json @default("{}")
max_parallel_requests Int?
metadata Json @default("{}")
@ -362,6 +457,7 @@ model LiteLLM_DeletedVerificationToken {
created_by String? // Original creator
updated_at DateTime? // Last update timestamp before deletion
updated_by String? // Last user who updated before deletion
last_active DateTime? // When this key was last used before deletion
rotation_count Int? @default(0)
auto_rotate Boolean? @default(false)
rotation_interval String?
@ -390,7 +486,9 @@ model LiteLLM_EndUserTable {
allowed_model_region String? // require all user requests to use models in this specific region
default_model String? // use along with 'allowed_model_region'. if no available model in region, default to this model.
budget_id String?
object_permission_id String?
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
blocked Boolean @default(false)
}
@ -432,7 +530,7 @@ model LiteLLM_SpendLogs {
custom_llm_provider String? @default("") // litellm used custom_llm_provider
api_base String? @default("")
user String? @default("")
metadata Json? @default("{}")
metadata Json? @default("{}") // project_id stored here
cache_hit String? @default("")
cache_key String? @default("")
request_tags Json? @default("[]")

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.39"
version = "0.4.44"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.39"
version = "0.4.44"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

55
litellm/__init__.py
View file

@ -1152,6 +1152,28 @@ from .skills.main import (
delete_skill,
adelete_skill,
)
from .evals.main import (
create_eval,
acreate_eval,
list_evals,
alist_evals,
get_eval,
aget_eval,
delete_eval,
adelete_eval,
cancel_eval,
acancel_eval,
create_run,
acreate_run,
list_runs,
alist_runs,
get_run,
aget_run,
delete_run,
adelete_run,
cancel_run,
acancel_run,
)
from .integrations import *
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
from .exceptions import (
@ -1333,6 +1355,7 @@ if TYPE_CHECKING:
from .llms.vertex_ai.rerank.transformation import VertexAIRerankConfig as VertexAIRerankConfig
from .llms.fireworks_ai.rerank.transformation import FireworksAIRerankConfig as FireworksAIRerankConfig
from .llms.voyage.rerank.transformation import VoyageRerankConfig as VoyageRerankConfig
from .llms.watsonx.rerank.transformation import IBMWatsonXRerankConfig as IBMWatsonXRerankConfig
from .llms.clarifai.chat.transformation import ClarifaiConfig as ClarifaiConfig
from .llms.ai21.chat.transformation import AI21ChatConfig as AI21ChatConfig
from .llms.meta_llama.chat.transformation import LlamaAPIConfig as LlamaAPIConfig
@ -1400,6 +1423,7 @@ if TYPE_CHECKING:
from .llms.volcengine.responses.transformation import VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig
from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig
from .llms.perplexity.responses.transformation import PerplexityResponsesConfig as PerplexityResponsesConfig
from .llms.databricks.responses.transformation import DatabricksResponsesAPIConfig as DatabricksResponsesAPIConfig
from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig
from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config
from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig
@ -1732,6 +1756,37 @@ def __getattr__(name: str) -> Any:
_globals["_service_logger"] = litellm._service_logger
return _globals["_service_logger"]
# Lazy load evals module functions
if name in ["acreate_eval", "alist_evals", "aget_eval", "aupdate_eval", "adelete_eval", "acancel_eval",
"create_eval", "list_evals", "get_eval", "update_eval", "delete_eval", "cancel_eval",
"acreate_run", "alist_runs", "aget_run", "acancel_run", "adelete_run",
"create_run", "list_runs", "get_run", "cancel_run", "delete_run"]:
from litellm.evals.main import (
acreate_eval,
alist_evals,
aget_eval,
aupdate_eval,
adelete_eval,
acancel_eval,
create_eval,
list_evals,
get_eval,
update_eval,
delete_eval,
cancel_eval,
acreate_run,
alist_runs,
aget_run,
acancel_run,
adelete_run,
create_run,
list_runs,
get_run,
cancel_run,
delete_run,
)
return locals()[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

7
litellm/_lazy_imports_registry.py
View file

@ -155,6 +155,7 @@ LLM_CONFIG_NAMES = (
"VertexAIRerankConfig",
"FireworksAIRerankConfig",
"VoyageRerankConfig",
"IBMWatsonXRerankConfig",
"ClarifaiConfig",
"AI21ChatConfig",
"LlamaAPIConfig",
@ -227,6 +228,7 @@ LLM_CONFIG_NAMES = (
"LiteLLMProxyResponsesAPIConfig",
"VolcEngineResponsesAPIConfig",
"PerplexityResponsesConfig",
"DatabricksResponsesAPIConfig",
"GoogleAIStudioInteractionsConfig",
"OpenAIOSeriesConfig",
"AnthropicSkillsConfig",
@ -671,6 +673,7 @@ _LLM_CONFIGS_IMPORT_MAP = {
"FireworksAIRerankConfig",
),
"VoyageRerankConfig": (".llms.voyage.rerank.transformation", "VoyageRerankConfig"),
"IBMWatsonXRerankConfig": (".llms.watsonx.rerank.transformation", "IBMWatsonXRerankConfig"),
"ClarifaiConfig": (".llms.clarifai.chat.transformation", "ClarifaiConfig"),
"AI21ChatConfig": (".llms.ai21.chat.transformation", "AI21ChatConfig"),
"LlamaAPIConfig": (".llms.meta_llama.chat.transformation", "LlamaAPIConfig"),
@ -906,6 +909,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.perplexity.responses.transformation",
"PerplexityResponsesConfig",
),
"DatabricksResponsesAPIConfig": (
".llms.databricks.responses.transformation",
"DatabricksResponsesAPIConfig",
),
"GoogleAIStudioInteractionsConfig": (
".llms.gemini.interactions.transformation",
"GoogleAIStudioInteractionsConfig",

4
litellm/_service_logger.py
View file

@ -312,10 +312,12 @@ class ServiceLogging(CustomLogger):
_duration, type(_duration)
)
) # invalid _duration value
# Batch polling callbacks (check_batch_cost) don't include call_type in kwargs.
# Use .get() to avoid KeyError.
await self.async_service_success_hook(
service=ServiceTypes.LITELLM,
duration=_duration,
call_type=kwargs["call_type"],
call_type=kwargs.get("call_type", "unknown")
)
except Exception as e:
raise e

32
litellm/anthropic_beta_headers_config.json
View file

@ -67,7 +67,7 @@
"compact-2026-01-12": null,
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": null,
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"effort-2025-11-24": null,
"fast-mode-2026-02-01": null,
@ -148,5 +148,35 @@
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": "web-search-2025-03-05"
},
"databricks": {
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": "compact-2026-01-12",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": "fast-mode-2026-02-01",
"files-api-2025-04-14": "files-api-2025-04-14",
"structured-output-2024-03-01": null,
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-servers-2025-12-04": null,
"oauth-2025-04-20": "oauth-2025-04-20",
"output-128k-2025-02-19": "output-128k-2025-02-19",
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
"web-search-2025-03-05": "web-search-2025-03-05"
}
}

38
litellm/batches/batch_utils.py
View file

@ -8,7 +8,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.types.llms.openai import Batch
from litellm.types.utils import CallTypes, ModelResponse, Usage
from litellm.types.utils import CallTypes, ModelInfo, ModelResponse, Usage
from litellm.utils import token_counter
@ -16,14 +16,22 @@ async def calculate_batch_cost_and_usage(
file_content_dictionary: List[dict],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
model_name: Optional[str] = None,
model_info: Optional[ModelInfo] = None,
) -> Tuple[float, Usage, List[str]]:
"""
Calculate the cost and usage of a batch
Calculate the cost and usage of a batch.
Args:
model_info: Optional deployment-level model info with custom batch
pricing. Threaded through to batch_cost_calculator so that
deployment-specific pricing (e.g. input_cost_per_token_batches)
is used instead of the global cost map.
"""
batch_cost = _batch_cost_calculator(
custom_llm_provider=custom_llm_provider,
file_content_dictionary=file_content_dictionary,
model_name=model_name,
model_info=model_info,
)
batch_usage = _get_batch_job_total_usage_from_file_content(
file_content_dictionary=file_content_dictionary,
@ -94,6 +102,7 @@ def _batch_cost_calculator(
file_content_dictionary: List[dict],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
model_name: Optional[str] = None,
model_info: Optional[ModelInfo] = None,
) -> float:
"""
Calculate the cost of a batch based on the output file id
@ -108,6 +117,7 @@ def _batch_cost_calculator(
total_cost = _get_batch_job_cost_from_file_content(
file_content_dictionary=file_content_dictionary,
custom_llm_provider=custom_llm_provider,
model_info=model_info,
)
verbose_logger.debug("total_cost=%s", total_cost)
return total_cost
@ -290,10 +300,13 @@ def _get_file_content_as_dictionary(file_content: bytes) -> List[dict]:
def _get_batch_job_cost_from_file_content(
file_content_dictionary: List[dict],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
model_info: Optional[ModelInfo] = None,
) -> float:
"""
Get the cost of a batch job from the file content
"""
from litellm.cost_calculator import batch_cost_calculator
try:
total_cost: float = 0.0
# parse the file content as json
@ -303,11 +316,22 @@ def _get_batch_job_cost_from_file_content(
for _item in file_content_dictionary:
if _batch_response_was_successful(_item):
_response_body = _get_response_from_batch_job_output_file(_item)
total_cost += litellm.completion_cost(
completion_response=_response_body,
custom_llm_provider=custom_llm_provider,
call_type=CallTypes.aretrieve_batch.value,
)
if model_info is not None:
usage = _get_batch_job_usage_from_response_body(_response_body)
model = _response_body.get("model", "")
prompt_cost, completion_cost = batch_cost_calculator(
usage=usage,
model=model,
custom_llm_provider=custom_llm_provider,
model_info=model_info,
)
total_cost += prompt_cost + completion_cost
else:
total_cost += litellm.completion_cost(
completion_response=_response_body,
custom_llm_provider=custom_llm_provider,
call_type=CallTypes.aretrieve_batch.value,
)
verbose_logger.debug("total_cost=%s", total_cost)
return total_cost
except Exception as e:

224
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -62,9 +62,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def __init__(self):
pass
def _handle_raw_dict_response_item(
self, item: Dict[str, Any], index: int
) -> Tuple[Optional[Any], int]:
def _handle_raw_dict_response_item(self, item: Dict[str, Any], index: int) -> Tuple[Optional[Any], int]:
"""
Handle raw dict response items from Responses API (e.g., GPT-5 Codex format).
@ -107,13 +105,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if item_type == "function_call":
# Extract provider_specific_fields if present and pass through as-is
provider_specific_fields = item.get("provider_specific_fields")
if provider_specific_fields and not isinstance(
provider_specific_fields, dict
):
if provider_specific_fields and not isinstance(provider_specific_fields, dict):
provider_specific_fields = (
dict(provider_specific_fields)
if hasattr(provider_specific_fields, "__dict__")
else {}
dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {}
)
tool_call_dict = {
@ -129,9 +123,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if provider_specific_fields:
tool_call_dict["provider_specific_fields"] = provider_specific_fields
# Also add to function's provider_specific_fields for consistency
tool_call_dict["function"][
"provider_specific_fields"
] = provider_specific_fields
tool_call_dict["function"]["provider_specific_fields"] = provider_specific_fields
msg = Message(
content=None,
@ -169,7 +161,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(
content, role # type: ignore
content,
role, # type: ignore
),
}
)
@ -186,7 +179,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
elif isinstance(content, list):
# Transform list content to Responses API format
tool_output = self._convert_content_to_responses_format(
content, "user" # Use "user" role to get input_* types
content,
"user", # Use "user" role to get input_* types
)
else:
# Fallback: convert unexpected types to input_text
@ -219,9 +213,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
{
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(
content, cast(str, role)
),
"content": self._convert_content_to_responses_format(content, cast(str, role)),
}
)
@ -344,9 +336,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
previous_response_id = optional_params.get("previous_response_id")
if previous_response_id:
# Use the existing session handler for responses API
verbose_logger.debug(
f"Chat provider: Warning ignoring previous response ID: {previous_response_id}"
)
verbose_logger.debug(f"Chat provider: Warning ignoring previous response ID: {previous_response_id}")
# Convert back to responses API format for the actual request
@ -368,9 +358,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"client": client,
}
verbose_logger.debug(
f"Chat provider: Final request model={api_model}, input_items={len(input_items)}"
)
verbose_logger.debug(f"Chat provider: Final request model={api_model}, input_items={len(input_items)}")
self._merge_responses_api_request_into_request_data(
request_data, responses_api_request, instructions
@ -450,9 +438,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
LiteLLMCompletionResponsesConfig,
)
tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call(
tool_call_item=item,
index=tool_call_index,
tool_call_dict = (
LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call(
tool_call_item=item,
index=tool_call_index,
)
)
accumulated_tool_calls.append(tool_call_dict)
tool_call_index += 1
@ -472,9 +462,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
tool_calls=accumulated_tool_calls,
reasoning_content=reasoning_content,
)
choices.append(
Choices(message=msg, finish_reason="tool_calls", index=index)
)
choices.append(Choices(message=msg, finish_reason="tool_calls", index=index))
reasoning_content = None
return choices
@ -510,17 +498,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
if len(choices) == 0:
if (
raw_response.incomplete_details is not None
and raw_response.incomplete_details.reason is not None
):
raise ValueError(
f"{model} unable to complete request: {raw_response.incomplete_details.reason}"
)
if raw_response.incomplete_details is not None and raw_response.incomplete_details.reason is not None:
raise ValueError(f"{model} unable to complete request: {raw_response.incomplete_details.reason}")
else:
raise ValueError(
f"Unknown items in responses API response: {raw_response.output}"
)
raise ValueError(f"Unknown items in responses API response: {raw_response.output}")
setattr(model_response, "choices", choices)
@ -529,11 +510,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
setattr(
model_response,
"usage",
ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
raw_response.usage
),
ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw_response.usage),
)
# Preserve hidden params from the ResponsesAPIResponse, especially the headers
# which contain important provider information like x-request-id
raw_response_hidden_params = getattr(raw_response, "_hidden_params", {})
@ -550,24 +529,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
model_response._hidden_params[key] = merged_headers
else:
model_response._hidden_params[key] = value
return model_response
def get_model_response_iterator(
self,
streaming_response: Union[
Iterator[str], AsyncIterator[str], "ModelResponse", "BaseModel"
],
streaming_response: Union[Iterator[str], AsyncIterator[str], "ModelResponse", "BaseModel"],
sync_stream: bool,
json_mode: Optional[bool] = False,
) -> BaseModelResponseIterator:
return OpenAiResponsesToChatCompletionStreamIterator(
streaming_response, sync_stream, json_mode
)
return OpenAiResponsesToChatCompletionStreamIterator(streaming_response, sync_stream, json_mode)
def _convert_content_str_to_input_text(
self, content: str, role: str
) -> Dict[str, Any]:
def _convert_content_str_to_input_text(self, content: str, role: str) -> Dict[str, Any]:
if role == "user" or role == "system" or role == "tool":
return {"type": "input_text", "text": content}
else:
@ -594,9 +567,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if actual_image_url is None:
raise ValueError(f"Invalid image URL: {content_image_url}")
image_param = ResponseInputImageParam(
image_url=actual_image_url, detail="auto", type="input_image"
)
image_param = ResponseInputImageParam(image_url=actual_image_url, detail="auto", type="input_image")
if detail:
image_param["detail"] = detail
@ -605,31 +576,29 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def _convert_content_to_responses_format(
self,
content: Union[
str,
Iterable[
Union["OpenAIMessageContentListBlock", "ChatCompletionThinkingBlock"]
],
content: Optional[
Union[
str,
Iterable[Union["OpenAIMessageContentListBlock", "ChatCompletionThinkingBlock"]],
]
],
role: str,
) -> List[Dict[str, Any]]:
"""Convert chat completion content to responses API format"""
from litellm.types.llms.openai import ChatCompletionImageObject
verbose_logger.debug(
f"Chat provider: Converting content to responses format - input type: {type(content)}"
)
verbose_logger.debug(f"Chat provider: Converting content to responses format - input type: {type(content)}")
if isinstance(content, str):
if content is None:
return [self._convert_content_str_to_input_text("", role)]
elif isinstance(content, str):
result = [self._convert_content_str_to_input_text(content, role)]
verbose_logger.debug(f"Chat provider: String content -> {result}")
return result
elif isinstance(content, list):
result = []
for i, item in enumerate(content):
verbose_logger.debug(
f"Chat provider: Processing content item {i}: {type(item)} = {item}"
)
verbose_logger.debug(f"Chat provider: Processing content item {i}: {type(item)} = {item}")
if isinstance(item, str):
converted = self._convert_content_str_to_input_text(item, role)
result.append(converted)
@ -638,9 +607,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Handle multimodal content
original_type = item.get("type")
if original_type == "text":
converted = self._convert_content_str_to_input_text(
item.get("text", ""), role
)
converted = self._convert_content_str_to_input_text(item.get("text", ""), role)
result.append(converted)
verbose_logger.debug(f"Chat provider: text -> {converted}")
elif original_type == "image_url":
@ -652,18 +619,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
),
)
result.append(converted)
verbose_logger.debug(
f"Chat provider: image_url -> {converted}"
)
verbose_logger.debug(f"Chat provider: image_url -> {converted}")
else:
# Try to map other types to responses API format
item_type = original_type or "input_text"
if item_type == "image":
converted = {"type": "input_image", **item}
result.append(converted)
verbose_logger.debug(
f"Chat provider: image -> {converted}"
)
verbose_logger.debug(f"Chat provider: image -> {converted}")
elif item_type in [
"input_text",
"input_image",
@ -675,18 +638,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
]:
# Already in responses API format
result.append(item)
verbose_logger.debug(
f"Chat provider: passthrough -> {item}"
)
verbose_logger.debug(f"Chat provider: passthrough -> {item}")
else:
# Default to input_text for unknown types
converted = self._convert_content_str_to_input_text(
str(item.get("text", item)), role
)
converted = self._convert_content_str_to_input_text(str(item.get("text", item)), role)
result.append(converted)
verbose_logger.debug(
f"Chat provider: unknown({original_type}) -> {converted}"
)
verbose_logger.debug(f"Chat provider: unknown({original_type}) -> {converted}")
verbose_logger.debug(f"Chat provider: Final converted content: {result}")
return result
else:
@ -694,17 +651,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
verbose_logger.debug(f"Chat provider: Other content type -> {result}")
return result
def _convert_tools_to_responses_format(
self, tools: List[Dict[str, Any]]
) -> List["ALL_RESPONSES_API_TOOL_PARAMS"]:
def _convert_tools_to_responses_format(self, tools: List[Dict[str, Any]]) -> List["ALL_RESPONSES_API_TOOL_PARAMS"]:
"""Convert chat completion tools to responses API tools format"""
responses_tools: List["ALL_RESPONSES_API_TOOL_PARAMS"] = []
for tool in tools:
# convert function tool from chat completion to responses API format
if tool.get("type") == "function":
function_tool = cast(
ChatCompletionToolParamFunctionChunk, tool.get("function")
)
function_tool = cast(ChatCompletionToolParamFunctionChunk, tool.get("function"))
responses_tools.append(
FunctionToolParam(
name=function_tool["name"],
@ -730,9 +683,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if not extra_body:
return optional_params
supported_responses_api_params = set(
ResponsesAPIOptionalRequestParams.__annotations__.keys()
)
supported_responses_api_params = set(ResponsesAPIOptionalRequestParams.__annotations__.keys())
# Also include params we handle specially
supported_responses_api_params.update(
{
@ -750,9 +701,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return optional_params
def _map_reasoning_effort(
self, reasoning_effort: Union[str, Dict[str, Any]]
) -> Optional[Reasoning]:
def _map_reasoning_effort(self, reasoning_effort: Union[str, Dict[str, Any]]) -> Optional[Reasoning]:
# If dict is passed, convert it directly to Reasoning object
if isinstance(reasoning_effort, dict):
return Reasoning(**reasoning_effort) # type: ignore[typeddict-item]
@ -760,8 +709,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Check if auto-summary is enabled via flag or environment variable
# Priority: litellm.reasoning_auto_summary flag > LITELLM_REASONING_AUTO_SUMMARY env var
auto_summary_enabled = (
litellm.reasoning_auto_summary
or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
)
# If string is passed, map with optional summary based on flag/env var
@ -772,11 +720,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
elif reasoning_effort == "xhigh":
return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh") # type: ignore[typeddict-item]
elif reasoning_effort == "medium":
return Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium")
return (
Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium")
)
elif reasoning_effort == "low":
return Reasoning(effort="low", summary="detailed") if auto_summary_enabled else Reasoning(effort="low")
elif reasoning_effort == "minimal":
return Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal")
return (
Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal")
)
return None
def _add_web_search_tool(
@ -855,7 +807,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return {"format": {"type": "text"}}
return None
@staticmethod
def _convert_annotations_to_chat_format(
annotations: Optional[List[Any]],
@ -908,9 +860,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
def __init__(
self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False
):
def __init__(self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False):
super().__init__(streaming_response, sync_stream, json_mode)
def _handle_string_chunk(
@ -923,9 +873,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
if not str_line or str_line.startswith("event:"):
# ignore.
return GenericStreamingChunk(
text="", tool_use=None, is_finished=False, finish_reason="", usage=None
)
return GenericStreamingChunk(text="", tool_use=None, is_finished=False, finish_reason="", usage=None)
index = str_line.find("data:")
if index != -1:
str_line = str_line[index + 5 :]
@ -988,13 +936,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
if output_item.get("type") == "function_call":
# Extract provider_specific_fields if present
provider_specific_fields = output_item.get("provider_specific_fields")
if provider_specific_fields and not isinstance(
provider_specific_fields, dict
):
if provider_specific_fields and not isinstance(provider_specific_fields, dict):
provider_specific_fields = (
dict(provider_specific_fields)
if hasattr(provider_specific_fields, "__dict__")
else {}
dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {}
)
function_chunk = ChatCompletionToolCallFunctionChunk(
@ -1003,9 +947,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
)
if provider_specific_fields:
function_chunk["provider_specific_fields"] = (
provider_specific_fields
)
function_chunk["provider_specific_fields"] = provider_specific_fields
tool_call_chunk = ChatCompletionToolCallChunk(
id=output_item.get("call_id"),
@ -1040,9 +982,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
id=None,
index=0,
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=None, arguments=content_part
),
function=ChatCompletionToolCallFunctionChunk(name=None, arguments=content_part),
)
]
),
@ -1051,22 +991,16 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
]
)
else:
raise ValueError(
f"Chat provider: Invalid function argument delta {parsed_chunk}"
)
raise ValueError(f"Chat provider: Invalid function argument delta {parsed_chunk}")
elif event_type == "response.output_item.done":
# New output item added
output_item = parsed_chunk.get("item", {})
if output_item.get("type") == "function_call":
# Extract provider_specific_fields if present
provider_specific_fields = output_item.get("provider_specific_fields")
if provider_specific_fields and not isinstance(
provider_specific_fields, dict
):
if provider_specific_fields and not isinstance(provider_specific_fields, dict):
provider_specific_fields = (
dict(provider_specific_fields)
if hasattr(provider_specific_fields, "__dict__")
else {}
dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {}
)
function_chunk = ChatCompletionToolCallFunctionChunk(
@ -1076,9 +1010,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
# Add provider_specific_fields to function if present
if provider_specific_fields:
function_chunk["provider_specific_fields"] = (
provider_specific_fields
)
function_chunk["provider_specific_fields"] = provider_specific_fields
tool_call_chunk = ChatCompletionToolCallChunk(
id=output_item.get("call_id"),
@ -1142,21 +1074,31 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
elif event_type == "response.completed":
# Response is fully complete - now we can signal is_finished=True
# This ensures we don't prematurely end the stream before tool_calls arrive
# Check if response contains function_call items in output
# to determine correct finish_reason
response_data = parsed_chunk.get("response", {})
output_items = response_data.get("output", []) if response_data else []
has_function_calls = any(
item.get("type") == "function_call" for item in output_items if isinstance(item, dict)
)
finish_reason = "tool_calls" if has_function_calls else "stop"
return ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(content=""),
finish_reason="stop",
finish_reason=finish_reason,
)
]
)
else:
pass
# For any unhandled event types, create a minimal valid chunk or skip
verbose_logger.debug(
f"Chat provider: Unhandled event type '{event_type}', creating empty chunk"
)
verbose_logger.debug(f"Chat provider: Unhandled event type '{event_type}', creating empty chunk")
# Return a minimal valid chunk for unknown events
return ModelResponseStream(
@ -1179,9 +1121,5 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
Returns:
ModelResponseStream: OpenAI-formatted streaming chunk
"""
verbose_logger.debug(
f"Chat provider: transform_streaming_response called with chunk: {chunk}"
)
return OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
chunk
)
verbose_logger.debug(f"Chat provider: transform_streaming_response called with chunk: {chunk}")
return OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(chunk)

45
litellm/constants.py
View file

@ -49,6 +49,19 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = int(
)
DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
# Maximum number of base64 characters to keep in logging payloads.
# Data URIs exceeding this are replaced with a size placeholder.
# Set to 0 to disable truncation.
MAX_BASE64_LENGTH_FOR_LOGGING = int(
os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64)
)
# When true, adds detailed per-phase timing breakdown headers to responses.
# Headers: x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms
LITELLM_DETAILED_TIMING = (
os.getenv("LITELLM_DETAILED_TIMING", "false").lower() == "true"
)
# Model cost map validation constants
MODEL_COST_MAP_MIN_MODEL_COUNT = int(
os.getenv("MODEL_COST_MAP_MIN_MODEL_COUNT", 50)
@ -91,6 +104,14 @@ MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH = int(
os.getenv("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150)
)
# Semantic Guard Defaults
DEFAULT_SEMANTIC_GUARD_EMBEDDING_MODEL = str(
os.getenv("DEFAULT_SEMANTIC_GUARD_EMBEDDING_MODEL", "text-embedding-3-small")
)
DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD = float(
os.getenv("DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD", 0.75)
)
# MCP OAuth2 Client Credentials Defaults
MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS = int(
os.getenv("MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS", "60")
@ -287,7 +308,9 @@ MIN_NON_ZERO_TEMPERATURE = float(os.getenv("MIN_NON_ZERO_TEMPERATURE", 0.0001))
REPEATED_STREAMING_CHUNK_LIMIT = int(
os.getenv("REPEATED_STREAMING_CHUNK_LIMIT", 100)
) # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives.
DEFAULT_MAX_LRU_CACHE_SIZE = int(os.getenv("DEFAULT_MAX_LRU_CACHE_SIZE", 16))
# Shared maxsize for functools.lru_cache usage across hot paths.
# Defaulted to 64 to avoid cache thrash in multi-model production workloads.
DEFAULT_MAX_LRU_CACHE_SIZE = int(os.getenv("DEFAULT_MAX_LRU_CACHE_SIZE", 64))
_REALTIME_BODY_CACHE_SIZE = 1000 # Keep realtime helper caches bounded; workloads rarely exceed 1k models/intents
INITIAL_RETRY_DELAY = float(os.getenv("INITIAL_RETRY_DELAY", 0.5))
MAX_RETRY_DELAY = float(os.getenv("MAX_RETRY_DELAY", 8.0))
@ -576,6 +599,10 @@ OPENAI_CHAT_COMPLETION_PARAMS = [
"thinking",
"web_search_options",
"service_tier",
"prompt_cache_key",
"prompt_cache_retention",
"safety_identifier",
"verbosity",
]
OPENAI_TRANSCRIPTION_PARAMS = [
@ -637,6 +664,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES = {
"prompt_cache_retention": None,
"store": None,
"metadata": None,
"context_management": None,
}
openai_compatible_endpoints: List = [
@ -1039,6 +1067,7 @@ BEDROCK_CONVERSE_MODELS = [
"anthropic.claude-sonnet-4-5-20250929-v1:0",
"anthropic.claude-opus-4-6-v1:0",
"anthropic.claude-opus-4-6-v1",
"anthropic.claude-sonnet-4-6",
"anthropic.claude-opus-4-1-20250805-v1:0",
"anthropic.claude-opus-4-20250514-v1:0",
"anthropic.claude-sonnet-4-20250514-v1:0",
@ -1261,6 +1290,9 @@ LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false"
LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int(
os.getenv("LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS", 86400)
) # 24 hours default
LITELLM_KEY_ROTATION_GRACE_PERIOD: str = os.getenv(
"LITELLM_KEY_ROTATION_GRACE_PERIOD", ""
) # Duration to keep old key valid after rotation (e.g. "24h", "2d"); empty = immediate revoke (default)
UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard"
LITELLM_PROXY_ADMIN_NAME = "default_user_id"
@ -1463,3 +1495,14 @@ MICROSOFT_USER_FIRST_NAME_ATTRIBUTE = str(
MICROSOFT_USER_LAST_NAME_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_LAST_NAME_ATTRIBUTE", "surname")
)
# Maximum payload size (in bytes) to fully serialize for DEBUG logging.
# Payloads larger than this are truncated to avoid multi-second json.dumps blocking the response.
MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG = int(
os.getenv("MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG", 102400)
) # 100 KB
# Policy template enrichment
MAX_COMPETITOR_NAMES = int(os.getenv("MAX_COMPETITOR_NAMES", 100))
COMPETITOR_LLM_TEMPERATURE = float(os.getenv("COMPETITOR_LLM_TEMPERATURE", 0.3))
DEFAULT_COMPETITOR_DISCOVERY_MODEL = "gpt-4o-mini"

27
litellm/cost_calculator.py
View file

@ -448,7 +448,9 @@ def cost_per_token( # noqa: PLR0915
elif custom_llm_provider == "anthropic":
return anthropic_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "bedrock":
return bedrock_cost_per_token(model=model, usage=usage_block)
return bedrock_cost_per_token(
model=model, usage=usage_block, service_tier=service_tier
)
elif custom_llm_provider == "openai":
return openai_cost_per_token(
model=model, usage=usage_block, service_tier=service_tier
@ -1896,9 +1898,16 @@ def batch_cost_calculator(
usage: Usage,
model: str,
custom_llm_provider: Optional[str] = None,
model_info: Optional[ModelInfo] = None,
) -> Tuple[float, float]:
"""
Calculate the cost of a batch job
Calculate the cost of a batch job.
Args:
model_info: Optional deployment-level model info containing custom
batch pricing (e.g. input_cost_per_token_batches). When provided,
skips the global litellm.get_model_info() lookup so that
deployment-specific pricing is used.
"""
_, custom_llm_provider, _, _ = litellm.get_llm_provider(
@ -1911,12 +1920,13 @@ def batch_cost_calculator(
custom_llm_provider,
)
try:
model_info: Optional[ModelInfo] = litellm.get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
except Exception:
model_info = None
if model_info is None:
try:
model_info = litellm.get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
except Exception:
model_info = None
if not model_info:
return 0.0, 0.0
@ -2138,4 +2148,3 @@ def handle_realtime_stream_cost_calculation(
return total_cost

33
litellm/evals/__init__.py Normal file
View file

@ -0,0 +1,33 @@
"""
Evals API operations
"""
from .main import (
acancel_eval,
acreate_eval,
adelete_eval,
aget_eval,
alist_evals,
aupdate_eval,
cancel_eval,
create_eval,
delete_eval,
get_eval,
list_evals,
update_eval,
)
__all__ = [
"acreate_eval",
"alist_evals",
"aget_eval",
"aupdate_eval",
"adelete_eval",
"acancel_eval",
"create_eval",
"list_evals",
"get_eval",
"update_eval",
"delete_eval",
"cancel_eval",
]

1944
litellm/evals/main.py Normal file
View file

File diff suppressed because it is too large Load diff

10
litellm/integrations/SlackAlerting/budget_alert_types.py
View file

@ -74,6 +74,14 @@ class ProjectedLimitExceededAlert(BaseBudgetAlertType):
return user_info.token or "default_id"
class ProjectBudgetAlert(BaseBudgetAlertType):
def get_event_message(self) -> str:
return "Project Budget: "
def get_id(self, user_info: CallInfo) -> str:
return user_info.token or "default_id"
def get_budget_alert_type(
type: Literal[
"token_budget",
@ -84,6 +92,7 @@ def get_budget_alert_type(
"organization_budget",
"proxy_budget",
"projected_limit_exceeded",
"project_budget",
],
) -> BaseBudgetAlertType:
"""Factory function to get the appropriate budget alert type class"""
@ -97,6 +106,7 @@ def get_budget_alert_type(
"organization_budget": OrganizationBudgetAlert(),
"token_budget": TokenBudgetAlert(),
"projected_limit_exceeded": ProjectedLimitExceededAlert(),
"project_budget": ProjectBudgetAlert(),
}
if type in alert_types:

9
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -538,6 +538,7 @@ class SlackAlerting(CustomBatchLogger):
"organization_budget",
"proxy_budget",
"projected_limit_exceeded",
"project_budget",
],
user_info: CallInfo,
):
@ -1378,9 +1379,13 @@ Model Info:
"""
if self.alerting is None:
return
# Start periodic flush if not already started
if not self.periodic_started and self.alerting is not None and len(self.alerting) > 0:
if (
not self.periodic_started
and self.alerting is not None
and len(self.alerting) > 0
):
asyncio.create_task(self.periodic_flush())
self.periodic_started = True

12
litellm/integrations/custom_guardrail.py
View file

@ -26,6 +26,7 @@ from litellm.types.utils import (
CallTypes,
GenericGuardrailAPIInputs,
GuardrailStatus,
GuardrailTracingDetail,
LLMResponseTypes,
StandardLoggingGuardrailInformation,
)
@ -520,9 +521,15 @@ class CustomGuardrail(CustomLogger):
masked_entity_count: Optional[Dict[str, int]] = None,
guardrail_provider: Optional[str] = None,
event_type: Optional[GuardrailEventHooks] = None,
tracing_detail: Optional[GuardrailTracingDetail] = None,
) -> None:
"""
Builds `StandardLoggingGuardrailInformation` and adds it to the request metadata so it can be used for logging to DataDog, Langfuse, etc.
Args:
tracing_detail: Optional typed dict with provider-specific tracing fields
(guardrail_id, policy_template, detection_method, confidence_score,
classification, match_details, patterns_checked, alert_recipients).
"""
if isinstance(guardrail_json_response, Exception):
guardrail_json_response = str(guardrail_json_response)
@ -559,6 +566,7 @@ class CustomGuardrail(CustomLogger):
end_time=end_time,
duration=duration,
masked_entity_count=masked_entity_count,
**(tracing_detail or {}),
)
def _append_guardrail_info(container: dict) -> None:
@ -814,8 +822,8 @@ def log_guardrail_information(func):
- during_call
- post_call
"""
import asyncio
import functools
import inspect
def _infer_event_type_from_function_name(
func_name: str,
@ -896,7 +904,7 @@ def log_guardrail_information(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
if asyncio.iscoroutinefunction(func):
if inspect.iscoroutinefunction(func):
return async_wrapper(*args, **kwargs)
return sync_wrapper(*args, **kwargs)

20
litellm/integrations/datadog/datadog_cost_management.py
View file

@ -93,7 +93,9 @@ class DatadogCostManagementLogger(CustomBatchLogger):
Aggregates costs by Provider, Model, and Date.
Returns a list of DatadogFOCUSCostEntry.
"""
aggregator: Dict[Tuple[str, str, str, Tuple[Tuple[str, str], ...]], DatadogFOCUSCostEntry] = {}
aggregator: Dict[
Tuple[str, str, str, Tuple[Tuple[str, str], ...]], DatadogFOCUSCostEntry
] = {}
for log in logs:
try:
@ -167,10 +169,20 @@ class DatadogCostManagementLogger(CustomBatchLogger):
metadata = log.get("metadata", {})
if metadata:
# Add user info
if "user_api_key_alias" in metadata:
# Add user info
if metadata.get("user_api_key_alias"):
tags["user"] = str(metadata["user_api_key_alias"])
if "user_api_key_team_alias" in metadata:
tags["team"] = str(metadata["user_api_key_team_alias"])
# Add Team Tag
team_tag = (
metadata.get("user_api_key_team_alias")
or metadata.get("team_alias") # type: ignore
or metadata.get("user_api_key_team_id")
or metadata.get("team_id") # type: ignore
)
if team_tag:
tags["team"] = str(team_tag)
# model_group is not in StandardLoggingMetadata TypedDict, so we need to access it via dict.get()
model_group = metadata.get("model_group") # type: ignore[misc]
if model_group:

11
litellm/integrations/datadog/datadog_handler.py
View file

@ -55,4 +55,15 @@ def get_datadog_tags(
request_tags = standard_logging_object.get("request_tags", []) or []
tags.extend(f"request_tag:{tag}" for tag in request_tags)
# Add Team Tag
metadata = standard_logging_object.get("metadata", {}) or {}
team_tag = (
metadata.get("user_api_key_team_alias")
or metadata.get("team_alias")
or metadata.get("user_api_key_team_id")
or metadata.get("team_id")
)
if team_tag:
tags.append(f"team:{team_tag}")
return ",".join(tags)

86
litellm/integrations/prometheus.py
View file

@ -22,6 +22,10 @@ from typing import (
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
get_litellm_metadata_from_kwargs,
get_metadata_variable_name_from_kwargs,
)
from litellm.proxy._types import (
LiteLLM_DeletedVerificationToken,
LiteLLM_TeamTable,
@ -1055,16 +1059,16 @@ class PrometheusLogger(CustomLogger):
enum_values=enum_values,
)
if (
standard_logging_payload["stream"] is True
): # log successful streaming requests from logging event hook.
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_proxy_total_requests_metric"
),
enum_values=enum_values,
)
self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
# increment litellm_proxy_total_requests_metric for all successful requests
# (both streaming and non-streaming) in this single location to prevent
# double-counting that occurs when async_post_call_success_hook also increments
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_proxy_total_requests_metric"
),
enum_values=enum_values,
)
self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
def _increment_token_metrics(
self,
@ -1086,13 +1090,6 @@ class PrometheusLogger(CustomLogger):
):
_tags = standard_logging_payload["request_tags"]
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_proxy_total_requests_metric"
),
enum_values=enum_values,
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_total_tokens_metric"
@ -1655,49 +1652,12 @@ class PrometheusLogger(CustomLogger):
):
"""
Proxy level tracking - triggered when the proxy responds with a success response to the client
Note: litellm_proxy_total_requests_metric is NOT incremented here to avoid
double-counting. It is incremented in async_log_success_event which fires
for all successful requests (both streaming and non-streaming).
"""
try:
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
)
if self._should_skip_metrics_for_invalid_key(
user_api_key_dict=user_api_key_dict
):
return
_metadata = data.get("metadata", {}) or {}
enum_values = UserAPIKeyLabelValues(
end_user=user_api_key_dict.end_user_id,
hashed_api_key=user_api_key_dict.api_key,
api_key_alias=user_api_key_dict.key_alias,
requested_model=data.get("model", ""),
team=user_api_key_dict.team_id,
team_alias=user_api_key_dict.team_alias,
user=user_api_key_dict.user_id,
user_email=user_api_key_dict.user_email,
status_code="200",
route=user_api_key_dict.request_route,
tags=StandardLoggingPayloadSetup._get_request_tags(
litellm_params=data,
proxy_server_request=data.get("proxy_server_request", {}),
),
client_ip=_metadata.get("requester_ip_address"),
user_agent=_metadata.get("user_agent"),
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_proxy_total_requests_metric"
),
enum_values=enum_values,
)
self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
except Exception as e:
verbose_logger.exception(
"prometheus Layer Error(): Exception occured - {}".format(str(e))
)
pass
pass
def _safe_get(self, obj: Any, key: str, default: Any = None) -> Any:
"""Get value from dict or Pydantic model."""
@ -2004,7 +1964,7 @@ class PrometheusLogger(CustomLogger):
api_base = standard_logging_payload["api_base"]
_litellm_params = request_kwargs.get("litellm_params", {}) or {}
_metadata = _litellm_params.get("metadata", {})
_metadata = get_litellm_metadata_from_kwargs(request_kwargs)
litellm_model_name = request_kwargs.get("model", None)
llm_provider = _litellm_params.get("custom_llm_provider", None)
_model_info = _metadata.get("model_info") or {}
@ -2220,7 +2180,8 @@ class PrometheusLogger(CustomLogger):
original_model_group,
kwargs,
)
_metadata = kwargs.get("metadata", {})
_metadata_key = get_metadata_variable_name_from_kwargs(kwargs)
_metadata = kwargs.get(_metadata_key) or {}
standard_metadata: StandardLoggingMetadata = (
StandardLoggingPayloadSetup.get_standard_logging_metadata(
metadata=_metadata
@ -2265,7 +2226,8 @@ class PrometheusLogger(CustomLogger):
kwargs,
)
_new_model = kwargs.get("model")
_metadata = kwargs.get("metadata", {})
_metadata_key = get_metadata_variable_name_from_kwargs(kwargs)
_metadata = kwargs.get(_metadata_key) or {}
_tags = cast(List[str], kwargs.get("tags") or [])
standard_metadata: StandardLoggingMetadata = (
StandardLoggingPayloadSetup.get_standard_logging_metadata(

83
litellm/integrations/s3_v2.py
View file

@ -51,6 +51,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_use_team_prefix: bool = False,
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
**kwargs,
):
try:
@ -78,7 +79,8 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_path=s3_path,
s3_use_team_prefix=s3_use_team_prefix,
s3_strip_base64_files=s3_strip_base64_files,
s3_use_key_prefix=s3_use_key_prefix
s3_use_key_prefix=s3_use_key_prefix,
s3_use_virtual_hosted_style=s3_use_virtual_hosted_style
)
verbose_logger.debug(f"s3 logger using endpoint url {s3_endpoint_url}")
@ -135,6 +137,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_use_team_prefix: bool = False,
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
):
"""
Initialize the s3 params for this logging callback
@ -217,6 +220,11 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
or s3_strip_base64_files
)
self.s3_use_virtual_hosted_style = (
bool(litellm.s3_callback_params.get("s3_use_virtual_hosted_style", False))
or s3_use_virtual_hosted_style
)
return
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
@ -247,8 +255,14 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
standard_logging_payload=kwargs.get("standard_logging_object", None),
)
# afile_delete and other non-model call types never produce a standard_logging_object,
# so s3_batch_logging_element is None. Skip gracefully instead of raising ValueError.
if s3_batch_logging_element is None:
raise ValueError("s3_batch_logging_element is None")
verbose_logger.debug(
"s3 Logging - skipping event, no standard_logging_object for call_type=%s",
kwargs.get("call_type", "unknown"),
)
return
verbose_logger.debug(
"\ns3 Logger - Logging payload = %s", s3_batch_logging_element
@ -302,13 +316,20 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
url = (
self.s3_endpoint_url
+ "/"
+ self.s3_bucket_name
+ "/"
+ batch_logging_element.s3_object_key
)
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = (
self.s3_endpoint_url
+ "/"
+ self.s3_bucket_name
+ "/"
+ batch_logging_element.s3_object_key
)
# Convert JSON to string
json_string = safe_dumps(batch_logging_element.payload)
@ -456,13 +477,20 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
url = (
self.s3_endpoint_url
+ "/"
+ self.s3_bucket_name
+ "/"
+ batch_logging_element.s3_object_key
)
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = (
self.s3_endpoint_url
+ "/"
+ self.s3_bucket_name
+ "/"
+ batch_logging_element.s3_object_key
)
# Convert JSON to string
json_string = safe_dumps(batch_logging_element.payload)
@ -550,13 +578,20 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
url = (
self.s3_endpoint_url
+ "/"
+ self.s3_bucket_name
+ "/"
+ s3_object_key
)
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = (
self.s3_endpoint_url
+ "/"
+ self.s3_bucket_name
+ "/"
+ s3_object_key
)
# Prepare the request for GET operation
# For GET requests, we need x-amz-content-sha256 with hash of empty string
@ -618,4 +653,4 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
verbose_logger.exception(
f"Error retrieving object {object_key} from cold storage: {str(e)}"
)
return None
return None

16
litellm/integrations/websearch_interception/handler.py
View file

@ -16,6 +16,7 @@ from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.websearch_interception.tools import (
get_litellm_web_search_tool,
get_litellm_web_search_tool_openai,
is_web_search_tool,
is_web_search_tool_chat_completion,
)
@ -77,7 +78,13 @@ class WebSearchInterceptionLogger(CustomLogger):
that we can intercept and execute ourselves.
"""
# Check if this is for an enabled provider
custom_llm_provider = kwargs.get("litellm_params", {}).get("custom_llm_provider", "")
# Try top-level kwargs first, then nested litellm_params, then derive from model name
custom_llm_provider = kwargs.get("custom_llm_provider", "") or kwargs.get("litellm_params", {}).get("custom_llm_provider", "")
if not custom_llm_provider:
try:
_, custom_llm_provider, _, _ = litellm.get_llm_provider(model=kwargs.get("model", ""))
except Exception:
custom_llm_provider = ""
if custom_llm_provider not in self.enabled_providers:
return None
@ -101,7 +108,7 @@ class WebSearchInterceptionLogger(CustomLogger):
for tool in tools:
if is_web_search_tool(tool):
# Convert to LiteLLM standard web search tool
converted_tool = get_litellm_web_search_tool()
converted_tool = get_litellm_web_search_tool_openai()
converted_tools.append(converted_tool)
verbose_logger.debug(
f"WebSearchInterception: Converted {tool.get('name', 'unknown')} "
@ -111,8 +118,9 @@ class WebSearchInterceptionLogger(CustomLogger):
# Keep other tools as-is
converted_tools.append(tool)
# Return modified kwargs with converted tools
return {"tools": converted_tools}
# Update tools in-place and return full kwargs
kwargs["tools"] = converted_tools
return kwargs
@classmethod
def from_config_yaml(

33
litellm/integrations/websearch_interception/tools.py
View file

@ -49,6 +49,39 @@ def get_litellm_web_search_tool() -> Dict[str, Any]:
}
def get_litellm_web_search_tool_openai() -> Dict[str, Any]:
"""
Get the standard LiteLLM web search tool definition in OpenAI format.
Used by async_pre_call_deployment_hook which runs in the chat completions
path where tools must be in OpenAI format (type: "function" with
function.parameters).
Returns:
Dict containing the OpenAI-style tool definition.
"""
return {
"type": "function",
"function": {
"name": LITELLM_WEB_SEARCH_TOOL_NAME,
"description": (
"Search the web for information. Use this when you need current "
"information or answers to questions that require up-to-date data."
),
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "The search query to execute"
}
},
"required": ["query"]
}
}
}
def is_web_search_tool_chat_completion(tool: Dict[str, Any]) -> bool:
"""
Check if a tool is a web search tool for Chat Completions API (strict check).

146
litellm/litellm_core_utils/litellm_logging.py
View file

@ -4,7 +4,6 @@
import copy
import datetime
import json
import logging
import os
import re
import subprocess
@ -65,6 +64,7 @@ from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
from litellm.litellm_core_utils.redact_messages import (
redact_message_input_output_from_custom_logger,
@ -335,7 +335,12 @@ class Logging(LiteLLMLoggingBaseClass):
messages = new_messages
self.model = model
self.messages = copy.deepcopy(messages) if messages is not None else None
# Shallow copy of the outer list only (inner message dicts are shared).
# Safe because the logging layer does not mutate individual message dicts.
_copy_start = time.time()
self.messages = copy.copy(messages) if messages is not None else None
self.message_copy_duration_ms: float = (time.time() - _copy_start) * 1000
self.callback_duration_ms: float = 0.0
self.stream = stream
self.start_time = start_time # log the call start time
self.call_type = call_type
@ -1630,15 +1635,26 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
] = self._build_standard_logging_payload(
logging_result, start_time, end_time
)
def _build_standard_logging_payload(
self, init_response_obj: Any, start_time: Any, end_time: Any
) -> Any:
"""Build StandardLoggingPayload and accumulate its construction time."""
_start = time.time()
payload = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=logging_result,
init_response_obj=init_response_obj,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
self.callback_duration_ms += (time.time() - _start) * 1000
return payload
def _transform_usage_objects(self, result):
if isinstance(result, ResponsesAPIResponse):
@ -1733,14 +1749,8 @@ class Logging(LiteLLMLoggingBaseClass):
elif isinstance(result, dict) or isinstance(result, list):
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
] = self._build_standard_logging_payload(
result, start_time, end_time
)
elif standard_logging_object is not None:
self.model_call_details[
@ -1912,14 +1922,8 @@ class Logging(LiteLLMLoggingBaseClass):
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
] = self._build_standard_logging_payload(
complete_streaming_response, start_time, end_time
)
if (
standard_logging_payload := self.model_call_details.get(
@ -2436,14 +2440,8 @@ class Logging(LiteLLMLoggingBaseClass):
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
] = self._build_standard_logging_payload(
complete_streaming_response, start_time, end_time
)
# print standard logging payload
@ -2466,14 +2464,8 @@ class Logging(LiteLLMLoggingBaseClass):
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
] = self._build_standard_logging_payload(
result, start_time, end_time
)
# print standard logging payload
@ -3132,75 +3124,9 @@ class Logging(LiteLLMLoggingBaseClass):
def get_combined_callback_list(
self, dynamic_success_callbacks: Optional[List], global_callbacks: List
) -> List:
# Combine dynamic and global callbacks
if dynamic_success_callbacks is None:
combined = list(global_callbacks)
else:
combined = list(dynamic_success_callbacks) + list(global_callbacks)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
"Combined callbacks BEFORE filtering: %s",
[self._get_callback_name(cb) for cb in combined],
)
# Filter duplicate Langfuse loggers to prevent trace leakage
# Only keep ONE Langfuse logger per request (prefer dynamic over global)
langfuse_logger_found = None
filtered = []
for cb in combined:
cb_name = self._get_callback_name(cb)
# Check if this is a Langfuse logger (vanilla or OTEL)
is_langfuse = cb_name.lower() in [
"langfuse",
"langfuselogger",
"langfuse_otel",
"langfuseotellogger",
]
if is_langfuse:
if langfuse_logger_found is None:
# First Langfuse logger found - keep it (dynamic has priority)
langfuse_logger_found = cb
filtered.append(cb)
verbose_logger.debug(
f"LiteLLM Logging: Using Langfuse logger: {cb_name} (other Langfuse loggers will be filtered out)"
)
else:
# Skip duplicate Langfuse logger to prevent trace leakage
verbose_logger.debug(
f"LiteLLM Logging: Skipping duplicate Langfuse logger: {cb_name} (already have {self._get_callback_name(langfuse_logger_found)})"
)
else:
# Keep all non-Langfuse callbacks
filtered.append(cb)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
"[LANGFUSE DEBUG] Filtered callbacks AFTER filtering: %s",
[self._get_callback_name(cb) for cb in filtered],
)
if langfuse_logger_found:
verbose_logger.debug(
"[LANGFUSE DEBUG] Langfuse logger kept: %s",
self._get_callback_name(langfuse_logger_found),
)
# After Langfuse filtering, deduplicate remaining callbacks
seen = set()
final = []
for cb in filtered:
cb_id = id(cb) if not isinstance(cb, str) else cb
if cb_id not in seen:
seen.add(cb_id)
final.append(cb)
else:
verbose_logger.debug(
f"LiteLLM Logging: Skipping duplicate callback: {self._get_callback_name(cb)}"
)
return final
return list(global_callbacks)
return list(set(dynamic_success_callbacks + global_callbacks))
def _remove_internal_litellm_callbacks(self, callbacks: List) -> List:
"""
@ -4573,6 +4499,7 @@ class StandardLoggingPayloadSetup:
user_api_key_budget_reset_at=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_project_id=None,
user_api_key_user_id=None,
user_api_key_team_alias=None,
user_api_key_user_email=None,
@ -4590,6 +4517,8 @@ class StandardLoggingPayloadSetup:
requester_custom_headers=None,
cold_storage_object_key=None,
user_api_key_auth_metadata=None,
team_alias=None,
team_id=None,
)
if isinstance(metadata, dict):
for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS:
@ -5277,8 +5206,10 @@ def get_standard_logging_object_payload(
model_id=_model_id,
requester_ip_address=clean_metadata.get("requester_ip_address", None),
user_agent=clean_metadata.get("user_agent", None),
messages=StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=kwargs, messages=kwargs.get("messages")
messages=truncate_base64_in_messages(
StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=kwargs, messages=kwargs.get("messages")
)
),
response=final_response_obj,
model_parameters=ModelParamHelper.get_standard_logging_model_parameters(
@ -5337,6 +5268,7 @@ def get_standard_logging_metadata(
user_api_key_budget_reset_at=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_project_id=None,
user_api_key_user_id=None,
user_api_key_user_email=None,
user_api_key_team_alias=None,
@ -5354,6 +5286,8 @@ def get_standard_logging_metadata(
user_api_key_request_route=None,
cold_storage_object_key=None,
user_api_key_auth_metadata=None,
team_alias=None,
team_id=None,
)
if isinstance(metadata, dict):
# Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields

2
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -602,7 +602,7 @@ def generic_cost_per_token( # noqa: PLR0915
total_details = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens
has_double_counting = cache_hit > 0 and total_details > usage.prompt_tokens
if text_tokens == 0 or has_double_counting:
if (text_tokens == 0 and prompt_tokens_details["image_count"] == 0) or has_double_counting:
text_tokens = (
usage.prompt_tokens
- cache_hit

6
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -546,7 +546,11 @@ def convert_to_model_response_object( # noqa: PLR0915
message = litellm.Message(content=json_mode_content_str)
finish_reason = "stop"
if message is None:
provider_specific_fields = {}
# Preserve provider_specific_fields if already present
# in the response (e.g. from proxy passthrough)
provider_specific_fields = dict(
choice["message"].get("provider_specific_fields", None) or {}
)
message_keys = Message.model_fields.keys()
for field in choice["message"].keys():
if field not in message_keys:

39
litellm/litellm_core_utils/llm_response_utils/response_metadata.py
View file

@ -1,6 +1,7 @@
import datetime
from typing import Any, Optional, Union
from litellm.constants import LITELLM_DETAILED_TIMING
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
from litellm.litellm_core_utils.logging_utils import LiteLLMLoggingObject
@ -108,7 +109,18 @@ class ResponseMetadata:
)
#########################################################
# 3. Add duration for reading from cache
# 3. Add callback processing duration
#########################################################
callback_duration_ms = getattr(logging_obj, "callback_duration_ms", None)
if callback_duration_ms is not None:
self._update_hidden_params(
{
"callback_duration_ms": round(callback_duration_ms, 4),
}
)
#########################################################
# 4. Add duration for reading from cache
# In this case overhead from litellm is the difference between the cache read duration and the total response time
#########################################################
if (
@ -128,6 +140,31 @@ class ResponseMetadata:
}
)
#########################################################
# 5. Detailed per-phase timing (opt-in via env var)
#########################################################
if LITELLM_DETAILED_TIMING and llm_api_duration_ms is not None:
detailed: dict = {
"timing_llm_api_ms": round(llm_api_duration_ms, 4),
}
# message copy time from Logging.__init__()
msg_copy_ms = getattr(logging_obj, "message_copy_duration_ms", None)
if msg_copy_ms is not None:
detailed["timing_message_copy_ms"] = round(msg_copy_ms, 4)
# pre-processing = time from request start to LLM API call start
api_call_start = logging_obj.model_call_details.get("api_call_start_time")
if api_call_start is not None and start_time is not None:
pre_ms = (api_call_start - start_time).total_seconds() * 1000
detailed["timing_pre_processing_ms"] = round(pre_ms, 4)
# post-processing = total - pre - llm_api
post_ms = total_response_time_ms - pre_ms - llm_api_duration_ms
detailed["timing_post_processing_ms"] = round(max(post_ms, 0), 4)
self._update_hidden_params(detailed)
def apply(self) -> None:
"""Apply metadata to the response object"""
if hasattr(self.result, "_hidden_params"):

109
litellm/litellm_core_utils/logging_utils.py
View file

@ -1,10 +1,13 @@
import asyncio
import functools
import inspect
import re
import time
from datetime import datetime
from typing import TYPE_CHECKING, Any, List, Optional, Union
from litellm._logging import verbose_logger
from litellm.constants import MAX_BASE64_LENGTH_FOR_LOGGING
from litellm.types.utils import (
ModelResponse,
ModelResponseStream,
@ -33,6 +36,110 @@ import litellm
Helper utils used for logging callbacks
"""
_BYTES_PER_KIB = 1024
_BYTES_PER_MIB = 1024 * 1024
# Regex matching data-URI base64 content: "data:<mime>;base64,<payload>"
# Captures: group(1)=mime_type, group(2)=base64_payload
_DATA_URI_RE = re.compile(r"data:([^;]+);base64,([A-Za-z0-9+/=]+)")
# Maximum nesting depth for _truncate_base64_in_value to guard against
# pathological payloads. OpenAI message format is typically 3-4 levels deep.
_MAX_TRUNCATION_DEPTH = 20
def _format_base64_size(num_chars: int) -> str:
"""Return a human-readable byte-size estimate from a base64 character count."""
num_bytes = num_chars * 3 / 4
if num_bytes >= _BYTES_PER_MIB:
return f"{num_bytes / _BYTES_PER_MIB:.2f}MB"
if num_bytes >= _BYTES_PER_KIB:
return f"{num_bytes / _BYTES_PER_KIB:.1f}KB"
return f"{int(num_bytes)}B"
def _base64_data_uri_replacer(match: re.Match) -> str:
"""Replace a single base64 data-URI match with a size placeholder if too long."""
mime_type = match.group(1)
payload = match.group(2)
if len(payload) <= MAX_BASE64_LENGTH_FOR_LOGGING:
return match.group(0)
size_str = _format_base64_size(len(payload))
return f"data:{mime_type};base64,[base64_data truncated: {size_str}]"
def _truncate_base64_in_string(value: str) -> str:
"""Replace long base64 data-URI payloads in a string with a size placeholder."""
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return value
return _DATA_URI_RE.sub(_base64_data_uri_replacer, value)
def _truncate_base64_in_value(value: Any) -> Any:
"""Iteratively truncate base64 data URIs in a JSON-like value (str/list/dict).
Uses an explicit stack instead of recursion to satisfy the project's
recursive-function detector and avoid stack-overflow on deep payloads.
"""
# Stack entries: (source_value, depth, parent_container, key_or_index)
# We mutate *copies* of dicts/lists in-place via parent references.
if isinstance(value, str):
return _truncate_base64_in_string(value)
if not isinstance(value, (dict, list)):
return value
# Shallow-copy the root so we don't mutate the caller's data.
root = {k: v for k, v in value.items()} if isinstance(value, dict) else list(value)
stack: list = [(root, 0)]
while stack:
container, depth = stack.pop()
if depth > _MAX_TRUNCATION_DEPTH:
continue
if isinstance(container, dict):
for k, v in container.items():
if isinstance(v, str):
container[k] = _truncate_base64_in_string(v)
elif isinstance(v, dict):
copy: Union[dict, list] = {ck: cv for ck, cv in v.items()}
container[k] = copy
stack.append((copy, depth + 1))
elif isinstance(v, list):
copy = list(v)
container[k] = copy
stack.append((copy, depth + 1))
elif isinstance(container, list):
for i, v in enumerate(container):
if isinstance(v, str):
container[i] = _truncate_base64_in_string(v)
elif isinstance(v, dict):
copy = {ck: cv for ck, cv in v.items()}
container[i] = copy
stack.append((copy, depth + 1))
elif isinstance(v, list):
copy = list(v)
container[i] = copy
stack.append((copy, depth + 1))
return root
def truncate_base64_in_messages(
messages: Optional[Union[str, list, dict]],
) -> Optional[Union[str, list, dict]]:
"""
Return a copy of *messages* with long base64 data-URI payloads replaced
by human-readable size placeholders.
"""
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return messages
try:
return _truncate_base64_in_value(messages)
except Exception as e:
verbose_logger.debug("Failed to truncate base64 in messages: %s", e)
return messages
# Global service logger instance to avoid recreating it
_service_logger = None
@ -270,7 +377,7 @@ def track_llm_api_timing():
verbose_logger.debug(f"Error in service logging: {str(e)}")
# Check if the function is async or sync
if asyncio.iscoroutinefunction(func):
if inspect.iscoroutinefunction(func):
return async_wrapper
return sync_wrapper

232
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -2018,6 +2018,235 @@ def anthropic_process_openai_file_message(
)
def _sanitize_empty_text_content(
message: AllMessageValues,
) -> AllMessageValues:
"""
Case C: Sanitize empty text content
- Replace empty or whitespace-only text content with a placeholder message.
Returns:
The message with sanitized content if needed, otherwise the original message
"""
if message.get("role") in ["user", "assistant"]:
content = message.get("content")
if isinstance(content, str):
if not content or not content.strip():
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = "[System: Empty message content sanitised to satisfy protocol]"
verbose_logger.debug(
f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message"
)
return message
def _add_missing_tool_results( # noqa: PLR0915
current_message: AllMessageValues,
messages: List[AllMessageValues],
current_index: int,
) -> Tuple[List[AllMessageValues], int]:
"""
Case A: Missing tool_result for tool_use (orphaned tool calls)
- If an assistant message has tool_calls but no corresponding tool result follows,
add a dummy tool result message indicating the user did not provide the result.
Returns:
A tuple of:
- List containing the assistant message, followed by existing tool results,
followed by any dummy tool results needed
- Number of original messages consumed (to adjust iteration index)
"""
result_messages: List[AllMessageValues] = []
tool_calls = current_message.get("tool_calls")
if not tool_calls or len(cast(list, tool_calls)) == 0:
return ([current_message], 0)
# Collect all tool_call_ids from this assistant message
expected_tool_call_ids = set()
for tool_call in cast(list, tool_calls):
tool_call_id = None
if isinstance(tool_call, dict):
tool_call_id = tool_call.get("id")
else:
tool_call_id = getattr(tool_call, "id", None)
if tool_call_id:
expected_tool_call_ids.add(tool_call_id)
# Collect actual tool result messages that follow this assistant message
found_tool_call_ids = set()
actual_tool_results: List[AllMessageValues] = []
j = current_index + 1
while j < len(messages):
next_msg = messages[j]
next_role = next_msg.get("role")
if next_role == "assistant":
break
if next_role in ["tool", "function"]:
tool_call_id = next_msg.get("tool_call_id")
if tool_call_id and tool_call_id in expected_tool_call_ids:
found_tool_call_ids.add(tool_call_id)
actual_tool_results.append(next_msg)
j += 1
# Find missing tool results
missing_tool_call_ids = expected_tool_call_ids - found_tool_call_ids
if missing_tool_call_ids:
verbose_logger.debug(
f"_add_missing_tool_results: Found {len(missing_tool_call_ids)} orphaned tool calls. Adding dummy tool results."
)
result_messages.append(current_message)
# Add existing tool results FIRST
result_messages.extend(actual_tool_results)
# Then add dummy tool results for missing ones
for tool_call_id in missing_tool_call_ids:
tool_name = "unknown_tool"
for tool_call in cast(list, tool_calls):
tc_id = None
if isinstance(tool_call, dict):
tc_id = tool_call.get("id")
else:
tc_id = getattr(tool_call, "id", None)
if tc_id == tool_call_id:
if isinstance(tool_call, dict):
function = tool_call.get("function", {})
if isinstance(function, dict):
tool_name = function.get("name", "unknown_tool")
else:
tool_name = getattr(function, "name", "unknown_tool")
else:
function = getattr(tool_call, "function", None)
if function:
tool_name = getattr(function, "name", "unknown_tool")
break
dummy_tool_result: ChatCompletionToolMessage = {
"role": "tool",
"tool_call_id": tool_call_id,
"content": f"[System: Tool execution skipped/interrupted by user. No result provided for tool '{tool_name}'.]",
}
result_messages.append(dummy_tool_result)
# Return the messages and the number of original messages to skip
return (result_messages, len(actual_tool_results))
return ([current_message], 0)
def _is_orphaned_tool_result(
current_message: AllMessageValues,
sanitized_messages: List[AllMessageValues],
) -> bool:
"""
Case B: Orphaned tool_result (unexpected result)
- Check if a tool message references a tool_call_id that doesn't exist in the previous
assistant message.
Returns:
True if this is an orphaned tool result that should be removed, False otherwise
"""
if current_message.get("role") not in ["tool", "function"]:
return False
tool_call_id = current_message.get("tool_call_id")
if not tool_call_id:
return False
# Look back to find the most recent assistant message with tool_calls
found_matching_tool_call = False
for j in range(len(sanitized_messages) - 1, -1, -1):
prev_msg = sanitized_messages[j]
if prev_msg.get("role") == "assistant":
tool_calls = prev_msg.get("tool_calls")
if tool_calls:
for tool_call in cast(list, tool_calls):
tc_id = None
if isinstance(tool_call, dict):
tc_id = tool_call.get("id")
else:
tc_id = getattr(tool_call, "id", None)
if tc_id == tool_call_id:
found_matching_tool_call = True
break
break
if not found_matching_tool_call:
verbose_logger.debug(
"_is_orphaned_tool_result: Found orphaned tool result with redacted tool_call_id"
)
return True
return False
def sanitize_messages_for_tool_calling(
messages: List[AllMessageValues],
) -> List[AllMessageValues]:
"""
Sanitize messages for tool calling to handle common issues when modify_params=True:
Case A: Missing tool_result for tool_use (orphaned tool calls)
- If an assistant message has tool_calls but no corresponding tool result follows,
add a dummy tool result message indicating the user did not provide the result.
Case B: Orphaned tool_result (unexpected result)
- If a tool message references a tool_call_id that doesn't exist in the previous
assistant message, remove that tool message.
Case C: Empty text content
- Replace empty or whitespace-only text content with a placeholder message.
This function operates on OpenAI format messages before they are converted to
provider-specific formats.
"""
if not litellm.modify_params:
return messages
sanitized_messages: List[AllMessageValues] = []
i = 0
while i < len(messages):
current_message = messages[i]
# Case C: Sanitize empty text content
current_message = _sanitize_empty_text_content(current_message)
# Case A: Check if assistant message has tool_calls without following tool results
if current_message.get("role") == "assistant":
result_messages, messages_consumed = _add_missing_tool_results(current_message, messages, i)
# If dummy tool results were added, extend sanitized_messages and skip consumed messages
if len(result_messages) > 1:
sanitized_messages.extend(result_messages)
# Skip the assistant message and any actual tool results that were included
i += 1 + messages_consumed
continue
# Case B: Check for orphaned tool results
if _is_orphaned_tool_result(current_message, sanitized_messages):
i += 1
continue # Skip this orphaned tool result
# Add the message to sanitized list
sanitized_messages.append(current_message)
i += 1
return sanitized_messages
def anthropic_messages_pt( # noqa: PLR0915
messages: List[AllMessageValues],
model: str,
@ -2037,6 +2266,9 @@ def anthropic_messages_pt( # noqa: PLR0915
5. System messages are a separate param to the Messages API
6. Ensure we only accept role, content. (message.name is not supported)
"""
# Sanitize messages for tool calling issues when modify_params=True
messages = sanitize_messages_for_tool_calling(messages)
# add role=tool support to allow function call result/error submission
user_message_types = {"user", "tool", "function"}
# reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them.

5
litellm/litellm_core_utils/redact_messages.py
View file

@ -9,6 +9,7 @@
import asyncio
import copy
import inspect
from typing import TYPE_CHECKING, Any, Optional
import litellm
@ -101,8 +102,8 @@ def perform_redaction(model_call_details: dict, result):
# Redact result
if result is not None:
# Check if result is a coroutine, async generator, or other async object - these cannot be deepcopied
if (asyncio.iscoroutine(result) or
asyncio.iscoroutinefunction(result) or
if (asyncio.iscoroutine(result) or
inspect.iscoroutinefunction(result) or
hasattr(result, '__aiter__') or # async generator
hasattr(result, '__anext__')): # async iterator
# For async objects, return a simple redacted response without deepcopy

78
litellm/litellm_core_utils/streaming_handler.py
View file

@ -2,6 +2,7 @@ import asyncio
import collections.abc
import datetime
import json
import logging
import threading
import time
import traceback
@ -435,7 +436,7 @@ class CustomStreamWrapper:
def handle_openai_chat_completion_chunk(self, chunk):
try:
print_verbose(f"\nRaw OpenAI Chunk\n{chunk}\n")
str_line = chunk
text = ""
is_finished = False
@ -485,7 +486,7 @@ class CustomStreamWrapper:
def handle_azure_text_completion_chunk(self, chunk):
try:
print_verbose(f"\nRaw OpenAI Chunk\n{chunk}\n")
text = ""
is_finished = False
finish_reason = None
@ -506,7 +507,7 @@ class CustomStreamWrapper:
def handle_openai_text_completion_chunk(self, chunk):
try:
print_verbose(f"\nRaw OpenAI Chunk\n{chunk}\n")
text = ""
is_finished = False
finish_reason = None
@ -870,9 +871,6 @@ class CustomStreamWrapper:
preserve_upstream_non_openai_attributes,
)
print_verbose(
f"completion_obj: {completion_obj}, model_response.choices[0]: {model_response.choices[0]}, response_obj: {response_obj}"
)
is_chunk_non_empty = self.is_chunk_non_empty(
completion_obj, model_response, response_obj
)
@ -899,11 +897,9 @@ class CustomStreamWrapper:
choice_json.pop(
"finish_reason", None
) # for mistral etc. which return a value in their last chunk (not-openai compatible).
print_verbose(f"choice_json: {choice_json}")
choices.append(StreamingChoices(**choice_json))
except Exception:
choices.append(StreamingChoices())
print_verbose(f"choices in streaming: {choices}")
setattr(model_response, "choices", choices)
else:
return
@ -921,9 +917,11 @@ class CustomStreamWrapper:
)
model_response = self.strip_role_from_delta(model_response)
verbose_logger.debug(
f"model_response.choices[0].delta inside is_chunk_non_empty: {model_response.choices[0].delta}"
)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
"model_response.choices[0].delta: %s",
model_response.choices[0].delta,
)
else:
## else
completion_obj["content"] = model_response_str
@ -1370,9 +1368,6 @@ class CustomStreamWrapper:
)
model_response.model = self.model
print_verbose(
f"model_response finish reason 3: {self.received_finish_reason}; response_obj={response_obj}"
)
## FUNCTION CALL PARSING
original_chunk = (
response_obj.get("original_chunk") if response_obj is not None else None
@ -1432,7 +1427,6 @@ class CustomStreamWrapper:
):
t.function.arguments = ""
_json_delta = delta.model_dump()
print_verbose(f"_json_delta: {_json_delta}")
if "role" not in _json_delta or _json_delta["role"] is None:
_json_delta[
"role"
@ -1466,11 +1460,7 @@ class CustomStreamWrapper:
if original_chunk.choices[0].delta is None
else dict(original_chunk.choices[0].delta)
)
print_verbose(f"original delta: {delta}")
model_response.choices[0].delta = Delta(**delta)
print_verbose(
f"new delta: {model_response.choices[0].delta}"
)
except Exception:
model_response.choices[0].delta = Delta()
else:
@ -1480,11 +1470,6 @@ class CustomStreamWrapper:
):
return model_response
return
print_verbose(
f"model_response.choices[0].delta: {model_response.choices[0].delta}; completion_obj: {completion_obj}"
)
print_verbose(f"self.sent_first_chunk: {self.sent_first_chunk}")
## CHECK FOR TOOL USE
if "tool_calls" in completion_obj and len(completion_obj["tool_calls"]) > 0:
@ -1915,18 +1900,9 @@ class CustomStreamWrapper:
and len(chunk.parts) == 0
):
continue
# chunk_creator() does logging/stream chunk building. We need to let it know its being called in_async_func, so we don't double add chunks.
# __anext__ also calls async_success_handler, which does logging
verbose_logger.debug(
f"PROCESSED ASYNC CHUNK PRE CHUNK CREATOR: {chunk}"
)
processed_chunk: Optional[ModelResponseStream] = self.chunk_creator(
chunk=chunk
)
verbose_logger.debug(
f"PROCESSED ASYNC CHUNK POST CHUNK CREATOR: {processed_chunk}"
)
if processed_chunk is None:
continue
@ -1943,31 +1919,33 @@ class CustomStreamWrapper:
self.rules.post_call_rules(
input=self.response_uptil_now, model=self.model
)
self.chunks.append(processed_chunk)
# Store a shallow copy so usage stripping below
# does not mutate the stored chunk.
self.chunks.append(processed_chunk.model_copy())
# Add mcp_list_tools to first chunk if present
if not self.sent_first_chunk:
processed_chunk = self._add_mcp_list_tools_to_first_chunk(processed_chunk)
self.sent_first_chunk = True
if hasattr(
processed_chunk, "usage"
): # remove usage from chunk, only send on final chunk
# Convert the object to a dictionary
if (
hasattr(processed_chunk, "usage")
and getattr(processed_chunk, "usage", None) is not None
):
# Strip usage from the outgoing chunk so it's not sent twice
# (once in the chunk, once in _hidden_params).
# Create a new object without usage, matching sync behavior.
# The copy in self.chunks retains usage for calculate_total_usage().
obj_dict = processed_chunk.model_dump()
# Remove an attribute (e.g., 'attr2')
if "usage" in obj_dict:
del obj_dict["usage"]
# Create a new object without the removed attribute
processed_chunk = self.model_response_creator(chunk=obj_dict)
processed_chunk = self.model_response_creator(
chunk=obj_dict, hidden_params=processed_chunk._hidden_params
)
is_empty = is_model_response_stream_empty(
model_response=cast(ModelResponseStream, processed_chunk)
)
if is_empty:
continue
print_verbose(f"final returned processed chunk: {processed_chunk}")
# add usage as hidden param
if self.sent_last_chunk is True and self.stream_options is None:
@ -1982,7 +1960,7 @@ class CustomStreamWrapper:
)
)
# Add MCP metadata to final chunk if present (after hooks)
processed_chunk = self._add_mcp_metadata_to_final_chunk(processed_chunk)
processed_chunk = self._add_mcp_metadata_to_final_chunk(processed_chunk) # type: ignore[reportArgumentType]
return processed_chunk
raise StopAsyncIteration
@ -1996,13 +1974,9 @@ class CustomStreamWrapper:
else:
chunk = next(self.completion_stream)
if chunk is not None and chunk != b"":
print_verbose(f"PROCESSED CHUNK PRE CHUNK CREATOR: {chunk}")
processed_chunk: Optional[
ModelResponseStream
] = self.chunk_creator(chunk=chunk)
print_verbose(
f"PROCESSED CHUNK POST CHUNK CREATOR: {processed_chunk}"
)
if processed_chunk is None:
continue
@ -2193,7 +2167,7 @@ def calculate_total_usage(chunks: List[ModelResponse]) -> Usage:
prompt_tokens: int = 0
completion_tokens: int = 0
for chunk in chunks:
if "usage" in chunk:
if "usage" in chunk and chunk["usage"] is not None:
if "prompt_tokens" in chunk["usage"]:
prompt_tokens = chunk["usage"].get("prompt_tokens", 0) or 0
if "completion_tokens" in chunk["usage"]:

5
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -124,6 +124,9 @@ class AnthropicMessagesHandler(BaseTranslation):
)
guardrailed_texts = guardrailed_inputs.get("texts", [])
guardrailed_tools = guardrailed_inputs.get("tools")
if guardrailed_tools is not None:
data["tools"] = guardrailed_tools
# Step 3: Map guardrail responses back to original message structure
await self._apply_guardrail_responses_to_input(
@ -194,7 +197,7 @@ class AnthropicMessagesHandler(BaseTranslation):
openai_tools = self.adapter.translate_anthropic_tools_to_openai(
tools=cast(List[AllAnthropicToolsValues], tools)
)
tools_to_check.extend(openai_tools)
tools_to_check.extend(openai_tools) # type: ignore
async def _apply_guardrail_responses_to_input(
self,

175
litellm/llms/anthropic/chat/transformation.py
View file

@ -171,9 +171,22 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return tool_call
@staticmethod
def _is_claude_opus_4_6(model: str) -> bool:
"""Check if the model is Claude Opus 4.5."""
return "opus-4-6" in model.lower() or "opus_4_6" in model.lower()
def _is_claude_4_6_model(model: str) -> bool:
"""Check if the model is a Claude 4.6 model that uses adaptive thinking."""
model_lower = model.lower()
return any(
model_variant in model_lower
for model_variant in (
"opus-4-6",
"opus_4_6",
"opus-4.6",
"opus_4.6",
"sonnet-4-6",
"sonnet_4_6",
"sonnet-4.6",
"sonnet_4.6",
)
)
def get_supported_openai_params(self, model: str):
params = [
@ -191,11 +204,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"user",
"web_search_options",
"speed",
"context_management",
]
if "claude-3-7-sonnet" in model or supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
if (
"claude-3-7-sonnet" in model
or AnthropicConfig._is_claude_4_6_model(model)
or supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
)
):
params.append("thinking")
params.append("reasoning_effort")
@ -206,27 +224,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
def filter_anthropic_output_schema(schema: Dict[str, Any]) -> Dict[str, Any]:
"""
Filter out unsupported fields from JSON schema for Anthropic's output_format API.
Anthropic's output_format doesn't support certain JSON schema properties:
- maxItems/minItems: Not supported for array types
- minimum/maximum: Not supported for numeric types
- minLength/maxLength: Not supported for string types
This mirrors the transformation done by the Anthropic Python SDK.
See: https://platform.claude.com/docs/en/build-with-claude/structured-outputs#how-sdk-transformation-works
The SDK approach:
1. Remove unsupported constraints from schema
2. Add constraint info to description (e.g., "Must be at least 100")
3. Validate responses against original schema
Args:
schema: The JSON schema dictionary to filter
Returns:
A new dictionary with unsupported fields removed and descriptions updated
Related issues:
Related issues:
- https://github.com/BerriAI/litellm/issues/19444
"""
if not isinstance(schema, dict):
@ -235,7 +252,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# All numeric/string/array constraints not supported by Anthropic
unsupported_fields = {
"maxItems", "minItems", # array constraints
"minimum", "maximum", # numeric constraints
"minimum", "maximum", # numeric constraints
"exclusiveMinimum", "exclusiveMaximum", # numeric constraints
"minLength", "maxLength", # string constraints
}
@ -705,12 +722,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
@staticmethod
def _map_reasoning_effort(
reasoning_effort: Optional[Union[REASONING_EFFORT, str]],
reasoning_effort: Optional[Union[REASONING_EFFORT, str]],
model: str,
) -> Optional[AnthropicThinkingParam]:
if reasoning_effort is None or reasoning_effort == "none":
return None
if AnthropicConfig._is_claude_opus_4_6(model):
if AnthropicConfig._is_claude_4_6_model(model):
return AnthropicThinkingParam(
type="adaptive",
)
@ -758,10 +775,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
if json_schema is None:
return None
# Filter out unsupported fields for Anthropic's output_format API
filtered_schema = self.filter_anthropic_output_schema(json_schema)
return AnthropicOutputSchema(
type="json_schema",
schema=filtered_schema,
@ -825,6 +842,62 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return hosted_web_search_tool
@staticmethod
def map_openai_context_management_to_anthropic(
context_management: Union[List[Dict[str, Any]], Dict[str, Any]]
) -> Optional[Dict[str, Any]]:
"""
OpenAI format: [{"type": "compaction", "compact_threshold": 200000}]
Anthropic format: {
"edits": [
{
"type": "compact_20260112",
"trigger": {"type": "input_tokens", "value": 150000}
}
]
}
Args:
context_management: OpenAI or Anthropic context_management parameter
Returns:
Anthropic-formatted context_management dict, or None if invalid
"""
# If already in Anthropic format (dict with 'edits'), pass through
if isinstance(context_management, dict) and "edits" in context_management:
return context_management
# If in OpenAI format (list), transform to Anthropic format
if isinstance(context_management, list):
anthropic_edits = []
for entry in context_management:
if not isinstance(entry, dict):
continue
entry_type = entry.get("type")
if entry_type == "compaction":
anthropic_edit: Dict[str, Any] = {
"type": "compact_20260112"
}
compact_threshold = entry.get("compact_threshold")
# Rewrite to 'trigger' with correct nesting if threshold exists
if compact_threshold is not None and isinstance(compact_threshold, (int, float)):
anthropic_edit["trigger"] = {
"type": "input_tokens",
"value": int(compact_threshold)
}
# Map any other keys by passthrough except handled ones
for k in entry:
if k not in {"type", "compact_threshold"}: # only passthrough other keys
anthropic_edit[k] = entry[k]
anthropic_edits.append(anthropic_edit)
if anthropic_edits:
return {"edits": anthropic_edits}
return None
def map_openai_params( # noqa: PLR0915
self,
non_default_params: dict,
@ -881,6 +954,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"opus-4-5",
"opus-4.6",
"opus-4-6",
"sonnet-4.6",
"sonnet-4-6",
"sonnet_4.6",
"sonnet_4_6",
}
):
_output_format = (
@ -927,9 +1004,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
elif param == "extra_headers":
optional_params["extra_headers"] = value
elif param == "context_management" and isinstance(value, dict):
# Pass through Anthropic-specific context_management parameter
optional_params["context_management"] = value
elif param == "context_management":
# Supports both OpenAI list format and Anthropic dict format
if isinstance(value, (list, dict)):
anthropic_context_management = self.map_openai_context_management_to_anthropic(value)
if anthropic_context_management is not None:
optional_params["context_management"] = anthropic_context_management
elif param == "speed" and isinstance(value, str):
# Pass through Anthropic-specific speed parameter for fast mode
optional_params["speed"] = value
@ -984,7 +1064,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_system_message_list: List[AnthropicSystemMessageContent] = []
for idx, message in enumerate(messages):
if message["role"] == "system":
valid_content: bool = False
system_prompt_indices.append(idx)
system_message_block = ChatCompletionSystemMessage(**message)
if isinstance(system_message_block["content"], str):
# Skip empty text blocks - Anthropic API raises errors for empty text
@ -1004,7 +1084,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_system_message_list.append(
anthropic_system_message_content
)
valid_content = True
elif isinstance(message["content"], list):
for _content in message["content"]:
# Skip empty text blocks - Anthropic API raises errors for empty text
@ -1028,10 +1107,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_system_message_list.append(
anthropic_system_message_content
)
valid_content = True
if valid_content:
system_prompt_indices.append(idx)
if len(system_prompt_indices) > 0:
for idx in reversed(system_prompt_indices):
messages.pop(idx)
@ -1076,7 +1152,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"""
Ensure a beta header value is present in the anthropic-beta header.
Merges with existing values instead of overriding them.
Args:
headers: Dictionary of headers to update
beta_value: The beta header value to add
@ -1090,32 +1166,39 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers["anthropic-beta"] = f"{existing_beta}, {beta_value}"
def _ensure_context_management_beta_header(
self, headers: dict, context_management: dict
self, headers: dict, context_management: object
) -> None:
"""
Add appropriate beta headers based on context_management edits.
- If any edit has type "compact_20260112", add compact-2026-01-12 header
- For all other edits, add context-management-2025-06-27 header
"""
edits = context_management.get("edits", [])
edits = []
# If anthropic format (dict with "edits" key)
if isinstance(context_management, dict) and "edits" in context_management:
edits = context_management.get("edits", [])
# If OpenAI format: list of context management entries
elif isinstance(context_management, list):
edits = context_management
# Defensive: ignore/fallback if context_management not valid
else:
return
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
if edit_type == "compact_20260112" or edit_type == "compaction":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
# Add compact header if any compact edits/entries exist
if has_compact:
self._ensure_beta_header(
headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value
)
# Add context management header if any other edits exist
# Add context management header if any other edits/entries exist
if has_other:
self._ensure_beta_header(
headers, ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
@ -1125,7 +1208,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
self, headers: dict, optional_params: dict
) -> dict:
"""Update headers with optional anthropic beta."""
# Skip adding beta headers for Vertex requests
# Vertex AI handles these headers differently
is_vertex_request = optional_params.get("is_vertex_request", False)
@ -1282,9 +1365,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
output_config = optional_params.get("output_config")
if output_config and isinstance(output_config, dict):
effort = output_config.get("effort")
if effort and effort not in ["high", "medium", "low"]:
if effort and effort not in ["high", "medium", "low", "max"]:
raise ValueError(
f"Invalid effort value: {effort}. Must be one of: 'high', 'medium', 'low'"
f"Invalid effort value: {effort}. Must be one of: 'high', 'medium', 'low', 'max'"
)
if effort == "max" and not self._is_claude_4_6_model(model):
raise ValueError(
f"effort='max' is only supported by Claude 4.6 models (Opus 4.6, Sonnet 4.6). Got model: {model}"
)
data["output_config"] = output_config
@ -1360,7 +1447,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
elif content["type"] == "web_fetch_tool_result":
if web_search_results is None:
web_search_results = []
web_search_results.append(content)
web_search_results.append(content)
else:
# All other tool results (bash_code_execution_tool_result, text_editor_code_execution_tool_result, etc.)
if tool_results is None:
@ -1377,7 +1464,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
thinking_blocks.append(
cast(ChatCompletionRedactedThinkingBlock, content)
)
## COMPACTION
elif content["type"] == "compaction":
if compaction_blocks is None:
@ -1585,7 +1672,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
provider_specific_fields["container"] = container
if compaction_blocks is not None:
provider_specific_fields["compaction_blocks"] = compaction_blocks
_message = litellm.Message(
tool_calls=tool_calls,
content=text_content or None,

Some files were not shown because too many files have changed in this diff Show more