scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-15 23:31:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 213

Merge branch 'main' into litellm_arize_ui

Browse source
This commit is contained in:
Mubashir Osmani 2025-10-16 11:24:53 -04:00 • committed by GitHub
commit a54445dc0e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
950 changed files with 62733 additions and 30792 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

118
.circleci/config.yml
View file

@ -616,6 +616,24 @@ jobs:
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Set DATABASE_URL environment variable
command: |
echo 'export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/circle_test"' >> $BASH_ENV
source $BASH_ENV
- run:
name: Run Security Scans
command: |
@ -658,18 +676,16 @@ jobs:
working_directory: ~/project
steps:
- checkout
- run:
name: Install PostgreSQL
command: |
sudo apt-get update
sudo apt-get install postgresql postgresql-contrib
echo 'export PATH=/usr/lib/postgresql/*/bin:$PATH' >> $BASH_ENV
- setup_google_dns
- run:
name: Show git commit hash
command: |
echo "Git commit hash: $CIRCLE_SHA1"
- run:
name: Install PostgreSQL
command: |
sudo apt-get update
sudo apt-get install -y postgresql-14 postgresql-contrib-14
- restore_cache:
keys:
- v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
@ -1025,6 +1041,49 @@ jobs:
paths:
- llm_responses_api_coverage.xml
- llm_responses_api_coverage
ocr_testing:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml ocr_coverage.xml
mv .coverage ocr_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- ocr_coverage.xml
- ocr_coverage
litellm_mapped_tests:
docker:
- image: cimg/python:3.11
@ -2357,6 +2416,25 @@ jobs:
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install "assemblyai==0.37.0"
- run:
name: Install dockerize
command: |
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@ -2367,10 +2445,11 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$CLEAN_STORE_MODEL_IN_DB_DATABASE_URL \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e STORE_MODEL_IN_DB="True" \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
my-app:latest \
@ -2400,7 +2479,16 @@ jobs:
python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
no_output_timeout:
120m
# Clean up first container
- run:
name: Stop and remove containers
command: |
docker stop my-app || true
docker rm my-app || true
docker stop postgres-db || true
docker rm postgres-db || true
when: always
- store_test_results:
path: test-results
proxy_build_from_pip_tests:
# Change from docker to machine executor
@ -2588,6 +2676,8 @@ jobs:
-e GEMINI_API_KEY=$GEMINI_API_KEY \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \
-e AZURE_API_KEY_PASSHROUGH=$AZURE_API_KEY_PASSHROUGH \
-e AZURE_API_BASE_PASSHROUGH=$AZURE_API_BASE_PASSHROUGH \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
@ -2694,7 +2784,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
coverage combine llm_translation_coverage llm_responses_api_coverage mcp_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage mcp_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@ -3242,6 +3332,12 @@ workflows:
only:
- main
- /litellm_.*/
- ocr_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_enterprise_tests:
filters:
branches:
@ -3291,6 +3387,7 @@ workflows:
- google_generate_content_endpoint_testing
- guardrails_testing
- llm_responses_api_testing
- ocr_testing
- litellm_mapped_tests
- litellm_mapped_enterprise_tests
- batches_testing
@ -3353,6 +3450,7 @@ workflows:
- mcp_testing
- google_generate_content_endpoint_testing
- llm_responses_api_testing
- ocr_testing
- litellm_mapped_tests
- litellm_mapped_enterprise_tests
- batches_testing

2
.gitignore vendored
View file

@ -97,3 +97,5 @@ litellm_config.yaml
.vscode/launch.json
litellm/proxy/to_delete_loadtest_work/*
update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
litellm/proxy/_experimental/out/guardrails/index.html

0
MCP_SSL_CHANGES_SUMMARY.md Normal file
View file

1
README.md
View file

@ -347,6 +347,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
| [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [Heroku](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | | | | |
| [OVHCloud AI Endpoints](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | | | | |
| [CometAPI](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
[**Read the Docs**](https://docs.litellm.ai/docs/)

47
ci_cd/security_scans.sh
View file

@ -68,8 +68,11 @@ run_grype_scans() {
# Allowlist of CVEs to be ignored in failure threshold/reporting
# - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
# - GHSA-4xh5-x5gv-qwph: GitHub Security Advisory alias for CVE-2025-8869
ALLOWED_CVES=(
"CVE-2025-8869"
"GHSA-4xh5-x5gv-qwph"
"CVE-2025-8291" # no fix available as of Oct 11, 2025
)
# Build JSON array of allowlisted CVE IDs for jq
@ -77,6 +80,26 @@ run_grype_scans() {
echo "Checking for vulnerabilities with CVSS score >= 4.0..."
echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
echo ""
# Show all high-severity vulnerabilities for transparency
TOTAL_HIGH_SEVERITY=$(grype litellm:latest -o json | jq -r '
.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| .vulnerability.id' | wc -l)
if [ "$TOTAL_HIGH_SEVERITY" -gt 0 ]; then
echo "Total vulnerabilities found with CVSS >= 4.0: $TOTAL_HIGH_SEVERITY"
echo ""
echo "All high-severity vulnerabilities (including allowlisted):"
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Allowlisted"],
(.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, (if (.vulnerability.id as $id | $allow | index($id)) then "YES" else "NO" end)])
| @tsv' | column -t -s $'\t'
echo ""
fi
HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
.matches[]
@ -85,8 +108,17 @@ run_grype_scans() {
| .vulnerability.id' | wc -l)
if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
echo "ERROR: Found $HIGH_SEVERITY_COUNT vulnerabilities with CVSS score >= 4.0 in litellm:latest"
echo ""
echo "=========================================="
echo "ERROR: Security Scan Failed"
echo "=========================================="
echo "Found $HIGH_SEVERITY_COUNT non-allowlisted vulnerabilities with CVSS score >= 4.0 in litellm:latest"
echo ""
echo "These vulnerabilities are NOT in the allowlist and must be addressed."
echo "Current allowlisted CVEs: ${ALLOWED_CVES[*]}"
echo ""
echo "Detailed vulnerability report:"
echo ""
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
(.matches[]
@ -94,6 +126,19 @@ run_grype_scans() {
| select((.vulnerability.id as $id | $allow | index($id) | not))
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
| @tsv' | column -t -s $'\t'
echo ""
echo "=========================================="
echo "Action Required:"
echo "=========================================="
echo "1. If a fix is available, update the package to the fixed version"
echo "2. If the vulnerability is not applicable or has no fix:"
echo " - Add the CVE/GHSA ID to ALLOWED_CVES array in ci_cd/security_scans.sh"
echo " - Add a comment explaining why it's safe to ignore"
echo ""
echo "Note: Some vulnerabilities may have multiple IDs (CVE-XXXX and GHSA-XXXX)."
echo "Add all relevant IDs to the allowlist if they refer to the same issue."
echo "=========================================="
echo ""
exit 1
else
echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"

474
cookbook/LiteLLM_CometAPI.ipynb vendored Normal file
View file

File diff suppressed because one or more lines are too long

2
docker-compose.yml
View file

@ -8,7 +8,7 @@ services:
#########################################
## Uncomment these lines to start proxy with a config.yaml file ##
# volumes:
# - ./config.yaml:/app/config.yaml <<- this is missing in the docker-compose file currently
# - ./config.yaml:/app/config.yaml
# command:
# - "--config=/app/config.yaml"
##############################################

2
docs/my-website/docs/adding_provider/new_rerank_provider.md
View file

@ -17,7 +17,7 @@ class YourProviderRerankConfig(BaseRerankConfig):
# ... other supported params
]
def transform_rerank_request(self, model: str, optional_rerank_params: OptionalRerankParams, headers: dict) -> dict:
def transform_rerank_request(self, model: str, optional_rerank_params: Dict, headers: dict) -> dict:
# Transform request to RerankRequest spec
return rerank_request.model_dump(exclude_none=True)

57
docs/my-website/docs/benchmarks.md
View file

@ -16,19 +16,17 @@ model_list:
api_key: "test"
```
### 1 Instance LiteLLM Proxy
### 2 Instance LiteLLM Proxy
In these tests the baseline latency characteristics are measured against a fake-openai-endpoint.
#### Performance Metrics
| Metric | Value |
|--------|-------|
| **Requests per Second (RPS)** | 475 |
| **End-to-End Latency P50 (ms)** | 100 |
| **LiteLLM Overhead P50 (ms)** | 3 |
| **LiteLLM Overhead P90 (ms)** | 17 |
| **LiteLLM Overhead P99 (ms)** | 31 |
| **Type** | **Name** | **Median (ms)** | **95%ile (ms)** | **99%ile (ms)** | **Average (ms)** | **Current RPS** |
| --- | --- | --- | --- | --- | --- | --- |
| POST | /chat/completions | 200 | 630 | 1200 | 262.46 | 1035.7 |
| Custom | LiteLLM Overhead Duration (ms) | 12 | 29 | 43 | 14.74 | 1035.7 |
| | Aggregated | 100 | 430 | 930 | 138.6 | 2071.4 |
<!-- <Image img={require('../img/1_instance_proxy.png')} /> -->
@ -36,28 +34,32 @@ In these tests the baseline latency characteristics are measured against a fake-
<Image img={require('../img/instances_vs_rps.png')} /> -->
### 4 Instances
| **Type** | **Name** | **Median (ms)** | **95%ile (ms)** | **99%ile (ms)** | **Average (ms)** | **Current RPS** |
| --- | --- | --- | --- | --- | --- | --- |
| POST | /chat/completions | 100 | 150 | 240 | 111.73 | 1170 |
| Custom | LiteLLM Overhead Duration (ms) | 2 | 8 | 13 | 3.32 | 1170 |
| | Aggregated | 77 | 130 | 180 | 57.53 | 2340 |
#### Key Findings
- Single instance: 475 RPS @ 100ms median latency
- LiteLLM adds 3ms P50 overhead, 17ms P90 overhead, 31ms P99 overhead
- 2 LiteLLM instances: 950 RPS @ 100ms latency
- 4 LiteLLM instances: 1900 RPS @ 100ms latency
### 2 Instances
**Adding 1 instance, will double the RPS and maintain the `100ms-110ms` median latency.**
| Metric | Litellm Proxy (2 Instances) |
|--------|------------------------|
| Median Latency (ms) | 100 |
| RPS | 950 |
- Doubling from 2 to 4 LiteLLM instances halves median latency: 200 ms → 100 ms.
- High-percentile latencies drop significantly: P95 630 ms → 150 ms, P99 1,200 ms → 240 ms.
- Setting workers equal to CPU count gives optimal performance.
## Machine Spec used for testing
Each machine deploying LiteLLM had the following specs:
- 2 CPU
- 4GB RAM
- 4 CPU
- 8GB RAM
## Locust Settings
- 1000 Users
- 500 user Ramp Up
## How to measure LiteLLM Overhead
@ -137,10 +139,3 @@ Using LangSmith has **no impact on latency, RPS compared to Basic Litellm Proxy*
|--------|------------------------|---------------------|
| RPS | 1133.2 | 1135 |
| Median Latency (ms) | 140 | 132 |
## Locust Settings
- 2500 Users
- 100 user Ramp Up

4
docs/my-website/docs/completion/input.md
View file

@ -180,11 +180,11 @@ def completion(
- `function`: *object* - Required.
- `tool_choice`: *string or object (optional)* - Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. Specifying a particular function via `{"type: "function", "function": {"name": "my_function"}}` forces the model to call that function.
- `tool_choice`: *string or object (optional)* - Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. Specifying a particular function via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that function.
- `none` is the default when no functions are present. `auto` is the default if functions are present.
- `parallel_tool_calls`: *boolean (optional)* - Whether to enable parallel function calling during tool use.. OpenAI default is true.
- `parallel_tool_calls`: *boolean (optional)* - Whether to enable parallel function calling during tool use. OpenAI default is true.
- `frequency_penalty`: *number or null (optional)* - It is used to penalize new tokens based on their frequency in the text so far.

52
docs/my-website/docs/embedding/supported_embedding.md
View file

@ -266,7 +266,59 @@ print(response)
| Titan Embeddings - G1 | `embedding(model="amazon.titan-embed-text-v1", input=input)` |
| Cohere Embeddings - English | `embedding(model="cohere.embed-english-v3", input=input)` |
| Cohere Embeddings - Multilingual | `embedding(model="cohere.embed-multilingual-v3", input=input)` |
| TwelveLabs Marengo (Async) | `embedding(model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text")` | [Async Invoke Docs](../providers/bedrock_embedding#async-invoke-embedding) |
## TwelveLabs Bedrock Embedding Models
TwelveLabs Marengo models support multimodal embeddings (text, image, video, audio) and require the `input_type` parameter to specify the input format.
### Usage
```python
from litellm import embedding
import os
# Set AWS credentials
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = "us-east-1"
# Text embedding
response = embedding(
model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0",
input=["Hello world from LiteLLM!"],
input_type="text" # Required parameter
)
# Image embedding (base64)
response = embedding(
model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
input=["data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..."],
input_type="image", # Required parameter
output_s3_uri="s3://your-bucket/async-invoke-output/"
)
# Video embedding (S3 URL)
response = embedding(
model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
input=["s3://your-bucket/video.mp4"],
input_type="video", # Required parameter
output_s3_uri="s3://your-bucket/async-invoke-output/"
)
```
### Required Parameters
| Parameter | Description | Values |
|-----------|-------------|--------|
| `input_type` | Type of input content | `"text"`, `"image"`, `"video"`, `"audio"` |
### Supported Models
| Model Name | Function Call | Notes |
|------------|---------------|-------|
| TwelveLabs Marengo 2.7 (Sync) | `embedding(model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text")` | Text embeddings only |
| TwelveLabs Marengo 2.7 (Async) | `embedding(model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text/image/video/audio")` | All input types, requires `output_s3_uri` |
## Cohere Embedding Models
https://docs.cohere.com/reference/embed

723
docs/my-website/docs/mcp.md
View file

@ -2,7 +2,7 @@ import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# /mcp - Model Context Protocol
# MCP Overview
LiteLLM Proxy provides an MCP Gateway that allows you to use a fixed endpoint for all MCP tools and control MCP access by Key, Team.
@ -23,6 +23,43 @@ LiteLLM Proxy provides an MCP Gateway that allows you to use a fixed endpoint fo
## Adding your MCP
### Prerequisites
To store MCP servers in the database, you need to enable database storage:
**Environment Variable:**
```bash
export STORE_MODEL_IN_DB=True
```
**OR in config.yaml:**
```yaml
general_settings:
store_model_in_db: true
```
#### Fine-grained Database Storage Control
By default, when `store_model_in_db` is `true`, all object types (models, MCPs, guardrails, vector stores, etc.) are stored in the database. If you want to store only specific object types, use the `supported_db_objects` setting.
**Example: Store only MCP servers in the database**
```yaml title="config.yaml" showLineNumbers
general_settings:
store_model_in_db: true
supported_db_objects: ["mcp"] # Only store MCP servers in DB
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-xxxxxxx
```
**See all available object types:** [Config Settings - supported_db_objects](./proxy/config_settings.md#general_settings---reference)
If `supported_db_objects` is not set, all object types are loaded from the database (default behavior).
<Tabs>
<TabItem value="ui" label="LiteLLM UI">
@ -209,8 +246,203 @@ litellm_settings:
</TabItem>
</Tabs>
## MCP Tool Filtering
## Converting OpenAPI Specs to MCP Servers
LiteLLM can automatically convert OpenAPI specifications into MCP servers, allowing you to expose any REST API as MCP tools. This is useful when you have existing APIs with OpenAPI/Swagger documentation and want to make them available as MCP tools.
### Benefits
- **Rapid Integration**: Convert existing APIs to MCP tools without writing custom MCP server code
- **Automatic Tool Generation**: LiteLLM automatically generates MCP tools from your OpenAPI spec
- **Unified Interface**: Use the same MCP interface for both native MCP servers and OpenAPI-based APIs
- **Easy Testing**: Test and iterate on API integrations quickly
### Configuration
Add your OpenAPI-based MCP server to your `config.yaml`:
```yaml title="config.yaml - OpenAPI to MCP" showLineNumbers
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-xxxxxxx
mcp_servers:
# OpenAPI Spec Example - Petstore API
petstore_mcp:
url: "https://petstore.swagger.io/v2"
spec_path: "/path/to/openapi.json"
auth_type: "none"
# OpenAPI Spec with API Key Authentication
my_api_mcp:
url: "http://0.0.0.0:8090"
spec_path: "/path/to/openapi.json"
auth_type: "api_key"
auth_value: "your-api-key-here"
# OpenAPI Spec with Bearer Token
secured_api_mcp:
url: "https://api.example.com"
spec_path: "/path/to/openapi.json"
auth_type: "bearer_token"
auth_value: "your-bearer-token"
```
### Configuration Parameters
| Parameter | Required | Description |
|-----------|----------|-------------|
| `url` | Yes | The base URL of your API endpoint |
| `spec_path` | Yes | Path or URL to your OpenAPI specification file (JSON or YAML) |
| `auth_type` | No | Authentication type: `none`, `api_key`, `bearer_token`, `basic`, `authorization` |
| `auth_value` | No | Authentication value (required if `auth_type` is set) |
| `description` | No | Optional description for the MCP server |
| `allowed_tools` | No | List of specific tools to allow (see [MCP Tool Filtering](#mcp-tool-filtering)) |
| `disallowed_tools` | No | List of specific tools to block (see [MCP Tool Filtering](#mcp-tool-filtering)) |
### Usage Example
Once configured, you can use the OpenAPI-based MCP server just like any other MCP server:
<Tabs>
<TabItem value="fastmcp" label="Python FastMCP">
```python title="Using OpenAPI-based MCP Server" showLineNumbers
from fastmcp import Client
import asyncio
# Standard MCP configuration
config = {
"mcpServers": {
"petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
}
}
}
# Create a client that connects to the server
client = Client(config)
async def main():
async with client:
# List available tools generated from OpenAPI spec
tools = await client.list_tools()
print(f"Available tools: {[tool.name for tool in tools]}")
# Example: Get a pet by ID (from Petstore API)
response = await client.call_tool(
name="getpetbyid",
arguments={"petId": "1"}
)
print(f"Response:\n{response}\n")
# Example: Find pets by status
response = await client.call_tool(
name="findpetsbystatus",
arguments={"status": "available"}
)
print(f"Response:\n{response}\n")
if __name__ == "__main__":
asyncio.run(main())
```
</TabItem>
<TabItem value="cursor" label="Cursor IDE">
```json title="Cursor MCP Configuration for OpenAPI Server" showLineNumbers
{
"mcpServers": {
"Petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY"
}
}
}
}
```
</TabItem>
<TabItem value="openai" label="OpenAI Responses API">
```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
curl --location 'https://api.openai.com/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer $OPENAI_API_KEY" \
--data '{
"model": "gpt-4o",
"tools": [
{
"type": "mcp",
"server_label": "petstore",
"server_url": "http://localhost:4000/petstore_mcp/mcp",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
}
}
],
"input": "Find all available pets in the petstore",
"tool_choice": "required"
}'
```
</TabItem>
</Tabs>
### How It Works
1. **Spec Loading**: LiteLLM loads your OpenAPI specification from the provided `spec_path`
2. **Tool Generation**: Each API endpoint in the spec becomes an MCP tool
3. **Parameter Mapping**: OpenAPI parameters are automatically mapped to MCP tool parameters
4. **Request Handling**: When a tool is called, LiteLLM converts the MCP request to the appropriate HTTP request
5. **Response Translation**: API responses are converted back to MCP format
### OpenAPI Spec Requirements
Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
- **Supported versions**: OpenAPI 3.0.x, OpenAPI 3.1.x, Swagger 2.0
- **Required fields**: `paths`, `info` sections should be properly defined
- **Operation IDs**: Each operation should have a unique `operationId` (this becomes the tool name)
- **Parameters**: Request parameters should be properly documented with types and descriptions
### Example OpenAPI Spec Structure
```yaml title="sample-openapi.yaml" showLineNumbers
openapi: 3.0.0
info:
title: My API
version: 1.0.0
paths:
/pets/{petId}:
get:
operationId: getPetById
summary: Get a pet by ID
parameters:
- name: petId
in: path
required: true
schema:
type: integer
responses:
'200':
description: Successful response
content:
application/json:
schema:
type: object
```
## Allow/Disallow MCP Tools
Control which tools are available from your MCP servers. You can either allow only specific tools or block dangerous ones.
<Tabs>
@ -269,210 +501,118 @@ mcp_servers:
- If you specify both `allowed_tools` and `disallowed_tools`, the allowed list takes priority
- Tool names are case-sensitive
## Using your MCP
---
### Use on LiteLLM UI
## Allow/Disallow MCP Tool Parameters
Follow this walkthrough to use your MCP on LiteLLM UI
Control which parameters are allowed for specific MCP tools using the `allowed_params` configuration. This provides fine-grained control over tool usage by restricting the parameters that can be passed to each tool.
<iframe width="840" height="500" src="https://www.loom.com/embed/57e0763267254bc79dbe6658d0b8758c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
### Configuration
### Use with Responses API
`allowed_params` is a dictionary that maps tool names to lists of allowed parameter names. When configured, only the specified parameters will be accepted for that tool - any other parameters will be rejected with a 403 error.
Replace `http://localhost:4000` with your LiteLLM Proxy base URL.
Demo Video Using Responses API with LiteLLM Proxy: [Demo video here](https://www.loom.com/share/34587e618c5c47c0b0d67b4e4d02718f?sid=2caf3d45-ead4-4490-bcc1-8d6dd6041c02)
<Tabs>
<TabItem value="curl" label="cURL">
```bash title="cURL Example" showLineNumbers
curl --location 'http://localhost:4000/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer sk-1234" \
--data '{
"model": "gpt-5",
"input": [
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
}
],
"tools": [
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never"
}
],
"stream": true,
"tool_choice": "required"
}'
```yaml title="config.yaml with allowed_params" showLineNumbers
mcp_servers:
deepwiki_mcp:
url: https://mcp.deepwiki.com/mcp
transport: "http"
auth_type: "none"
allowed_params:
# Tool name: list of allowed parameters
read_wiki_contents: ["status"]
my_api_mcp:
url: "https://my-api-server.com"
auth_type: "api_key"
auth_value: "my-key"
allowed_params:
# Using unprefixed tool name
getpetbyid: ["status"]
# Using prefixed tool name (both formats work)
my_api_mcp-findpetsbystatus: ["status", "limit"]
# Another tool with multiple allowed params
create_issue: ["title", "body", "labels"]
```
</TabItem>
<TabItem value="python" label="Python SDK">
### How It Works
```python title="Python SDK Example" showLineNumbers
"""
Use LiteLLM Proxy MCP Gateway to call MCP tools.
1. **Tool-specific filtering**: Each tool can have its own list of allowed parameters
2. **Flexible naming**: Tool names can be specified with or without the server prefix (e.g., both `"getpetbyid"` and `"my_api_mcp-getpetbyid"` work)
3. **Whitelist approach**: Only parameters in the allowed list are permitted
4. **Unlisted tools**: If `allowed_params` is not set, all parameters are allowed
5. **Error handling**: Requests with disallowed parameters receive a 403 error with details about which parameters are allowed
When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
"""
import openai
### Example Request Behavior
client = openai.OpenAI(
api_key="sk-1234", # paste your litellm proxy api key here
base_url="http://localhost:4000" # paste your litellm proxy base url here
)
print("Making API request to Responses API with MCP tools")
With the configuration above, here's how requests would be handled:
response = client.responses.create(
model="gpt-5",
input=[
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
}
],
tools=[
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never"
}
],
stream=True,
tool_choice="required"
)
for chunk in response:
print("response chunk: ", chunk)
```
</TabItem>
</Tabs>
#### Specifying MCP Tools
You can specify which MCP tools are available by using the `allowed_tools` parameter. This allows you to restrict access to specific tools within an MCP server.
To get the list of allowed tools when using LiteLLM MCP Gateway, you can naigate to the LiteLLM UI on MCP Servers > MCP Tools > Click the Tool > Copy Tool Name.
<Tabs>
<TabItem value="curl" label="cURL">
```bash title="cURL Example with allowed_tools" showLineNumbers
curl --location 'http://localhost:4000/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer sk-1234" \
--data '{
"model": "gpt-5",
"input": [
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
}
],
"tools": [
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy/mcp",
"require_approval": "never",
"allowed_tools": ["GitMCP-fetch_litellm_documentation"]
}
],
"stream": true,
"tool_choice": "required"
}'
```
</TabItem>
<TabItem value="python" label="Python SDK">
```python title="Python SDK Example with allowed_tools" showLineNumbers
import openai
client = openai.OpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
response = client.responses.create(
model="gpt-5",
input=[
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
}
],
tools=[
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy/mcp",
"require_approval": "never",
"allowed_tools": ["GitMCP-fetch_litellm_documentation"]
}
],
stream=True,
tool_choice="required"
)
print(response)
```
</TabItem>
</Tabs>
### Use with Cursor IDE
Use tools directly from Cursor IDE with LiteLLM MCP:
**Setup Instructions:**
1. **Open Cursor Settings**: Use `⇧+⌘+J` (Mac) or `Ctrl+Shift+J` (Windows/Linux)
2. **Navigate to MCP Tools**: Go to the "MCP Tools" tab and click "New MCP Server"
3. **Add Configuration**: Copy and paste the JSON configuration below, then save with `Cmd+S` or `Ctrl+S`
```json title="Basic Cursor MCP Configuration" showLineNumbers
**✅ Allowed Request:**
```json
{
"mcpServers": {
"LiteLLM": {
"url": "litellm_proxy",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY"
}
}
"tool": "read_wiki_contents",
"arguments": {
"status": "active"
}
}
```
#### How it works when server_url="litellm_proxy"
**❌ Rejected Request:**
```json
{
"tool": "read_wiki_contents",
"arguments": {
"status": "active",
"limit": 10 // This parameter is not allowed
}
}
```
When server_url="litellm_proxy", LiteLLM bridges non-MCP providers to your MCP tools.
**Error Response:**
```json
{
"error": "Parameters ['limit'] are not allowed for tool read_wiki_contents. Allowed parameters: ['status']. Contact proxy admin to allow these parameters."
}
```
- Tool Discovery: LiteLLM fetches MCP tools and converts them to OpenAI-compatible definitions
- LLM Call: Tools are sent to the LLM with your input; LLM selects which tools to call
- Tool Execution: LiteLLM automatically parses arguments, routes calls to MCP servers, executes tools, and retrieves results
- Response Integration: Tool results are sent back to LLM for final response generation
- Output: Complete response combining LLM reasoning with tool execution results
### Use Cases
This enables MCP tool usage with any LiteLLM-supported provider, regardless of native MCP support.
- **Security**: Prevent users from accessing sensitive parameters or dangerous operations
- **Cost control**: Restrict expensive parameters (e.g., limiting result counts)
- **Compliance**: Enforce parameter usage policies for regulatory requirements
- **Staged rollouts**: Gradually enable parameters as tools are tested
- **Multi-tenant isolation**: Different parameter access for different user groups
#### Auto-execution for require_approval: "never"
### Combining with Tool Filtering
Setting require_approval: "never" triggers automatic tool execution, returning the final response in a single API call without additional user interaction.
`allowed_params` works alongside `allowed_tools` and `disallowed_tools` for complete control:
```yaml title="Combined filtering example" showLineNumbers
mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
auth_type: oauth2
authorization_url: https://github.com/login/oauth/authorize
token_url: https://github.com/login/oauth/access_token
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
scopes: ["public_repo", "user:email"]
# Only allow specific tools
allowed_tools: ["create_issue", "list_issues", "search_issues"]
# Block dangerous operations
disallowed_tools: ["delete_repo"]
# Restrict parameters per tool
allowed_params:
create_issue: ["title", "body", "labels"]
list_issues: ["state", "sort", "perPage"]
search_issues: ["query", "sort", "order", "perPage"]
```
This configuration ensures that:
1. Only the three listed tools are available
2. The `delete_repo` tool is explicitly blocked
3. Each tool can only use its specified parameters
---
## MCP Server Access Control
@ -1064,6 +1204,8 @@ mcp_servers:
scopes: ["public_repo", "user:email"]
```
[**See Claude Code Tutorial**](./tutorials/claude_responses_api#connecting-mcp-servers)
## Using your MCP with client side credentials
Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.
@ -1452,221 +1594,6 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
}'
```
## MCP Cost Tracking
LiteLLM provides two ways to track costs for MCP tool calls:
| Method | When to Use | What It Does |
|--------|-------------|--------------|
| **Config-based Cost Tracking** | Simple cost tracking with fixed costs per tool/server | Automatically tracks costs based on configuration |
| **Custom Post-MCP Hook** | Dynamic cost tracking with custom logic | Allows custom cost calculations and response modifications |
### Config-based Cost Tracking
Configure fixed costs for MCP servers directly in your config.yaml:
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-xxxxxxx
mcp_servers:
zapier_server:
url: "https://actions.zapier.com/mcp/sk-xxxxx/sse"
mcp_info:
mcp_server_cost_info:
# Default cost for all tools in this server
default_cost_per_query: 0.01
# Custom cost for specific tools
tool_name_to_cost_per_query:
send_email: 0.05
create_document: 0.03
expensive_api_server:
url: "https://api.expensive-service.com/mcp"
mcp_info:
mcp_server_cost_info:
default_cost_per_query: 1.50
```
### Custom Post-MCP Hook
Use this when you need dynamic cost calculation or want to modify the MCP response before it's returned to the user.
#### 1. Create a custom MCP hook file
```python title="custom_mcp_hook.py" showLineNumbers
from typing import Optional
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.mcp import MCPPostCallResponseObject
class CustomMCPCostTracker(CustomLogger):
"""
Custom handler for MCP cost tracking and response modification
"""
async def async_post_mcp_tool_call_hook(
self,
kwargs,
response_obj: MCPPostCallResponseObject,
start_time,
end_time
) -> Optional[MCPPostCallResponseObject]:
"""
Called after each MCP tool call.
Modify costs and response before returning to user.
"""
# Extract tool information from kwargs
tool_name = kwargs.get("name", "")
server_name = kwargs.get("server_name", "")
# Calculate custom cost based on your logic
custom_cost = 42.00
# Set the response cost
response_obj.hidden_params.response_cost = custom_cost
return response_obj
# Create instance for LiteLLM to use
custom_mcp_cost_tracker = CustomMCPCostTracker()
```
#### 2. Configure in config.yaml
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-xxxxxxx
# Add your custom MCP hook
callbacks:
- custom_mcp_hook.custom_mcp_cost_tracker
mcp_servers:
zapier_server:
url: "https://actions.zapier.com/mcp/sk-xxxxx/sse"
```
#### 3. Start the proxy
```shell
$ litellm --config /path/to/config.yaml
```
When MCP tools are called, your custom hook will:
1. Calculate costs based on your custom logic
2. Modify the response if needed
3. Track costs in LiteLLM's logging system
## MCP Guardrails
LiteLLM supports applying guardrails to MCP tool calls to ensure security and compliance. You can configure guardrails to run before or during MCP calls to validate inputs and block or mask sensitive information.
### Supported MCP Guardrail Modes
MCP guardrails support the following modes:
- `pre_mcp_call`: Run **before** MCP call, on **input**. Use this mode when you want to apply validation/masking/blocking for MCP requests
- `during_mcp_call`: Run **during** MCP call execution. Use this mode for real-time monitoring and intervention
### Configuration Examples
Configure guardrails to run before MCP tool calls to validate and sanitize inputs:
```yaml title="config.yaml" showLineNumbers
guardrails:
- guardrail_name: "mcp-input-validation"
litellm_params:
guardrail: presidio # or other supported guardrails
mode: "pre_mcp_call" # or during_mcp_call
pii_entities_config:
CREDIT_CARD: "BLOCK" # Will block requests containing credit card numbers
EMAIL_ADDRESS: "MASK" # Will mask email addresses
PHONE_NUMBER: "MASK" # Will mask phone numbers
default_on: true
```
### Usage Examples
#### Testing Pre-MCP Call Guardrails
Test your MCP guardrails with a request that includes sensitive information:
```bash title="Test MCP Guardrail" showLineNumbers
curl http://localhost:4000/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{"role": "user", "content": "My credit card is 4111-1111-1111-1111 and my email is john@example.com"}
],
"guardrails": ["mcp-input-validation"]
}'
```
The request will be processed as follows:
1. Credit card number will be blocked (request rejected)
2. Email address will be masked (e.g., replaced with `<EMAIL_ADDRESS>`)
#### Using with MCP Tools
When using MCP tools, guardrails will be applied to the tool inputs:
```python title="Python Example with MCP Guardrails" showLineNumbers
import openai
client = openai.OpenAI(
api_key="your-api-key",
base_url="http://localhost:4000"
)
# This request will trigger MCP guardrails
response = client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "user", "content": "Send an email to 555-123-4567 with my SSN 123-45-6789"}
],
tools=[{"type": "mcp", "server_label": "litellm", "server_url": "litellm_proxy"}],
guardrails=["mcp-input-validation"]
)
```
### Supported Guardrail Providers
MCP guardrails work with all LiteLLM-supported guardrail providers:
- **Presidio**: PII detection and masking
- **Bedrock**: AWS Bedrock guardrails
- **Lakera**: Content moderation
- **Aporia**: Custom guardrails
- **Custom**: Your own guardrail implementations
## MCP Permission Management
LiteLLM supports managing permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. When a MCP client attempts to list tools, LiteLLM will only return the tools the entity has permissions to access.
When Creating a Key, Team, or Organization, you can select the allowed MCP Servers that the entity has access to.
<Image
img={require('../img/mcp_key.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
## LiteLLM Proxy - Walk through MCP Gateway
LiteLLM exposes an MCP Gateway for admins to add all their MCP servers to LiteLLM. The key benefits of using LiteLLM Proxy with MCP are:

45
docs/my-website/docs/mcp_control.md Normal file
View file

@ -0,0 +1,45 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# MCP Permission Management
Control which MCP servers and tools can be accessed by specific keys, teams, or organizations in LiteLLM. When a client attempts to list or call tools, LiteLLM enforces access controls based on configured permissions.
## Overview
LiteLLM provides fine-grained permission management for MCP servers, allowing you to:
- **Restrict MCP access by entity**: Control which keys, teams, or organizations can access specific MCP servers
- **Tool-level filtering**: Automatically filter available tools based on entity permissions
- **Centralized control**: Manage all MCP permissions from the LiteLLM Admin UI or API
This ensures that only authorized entities can discover and use MCP tools, providing an additional security layer for your MCP infrastructure.
:::info Related Documentation
- [MCP Overview](./mcp.md) - Learn about MCP in LiteLLM
- [MCP Cost Tracking](./mcp_cost.md) - Track costs for MCP tool calls
- [MCP Guardrails](./mcp_guardrail.md) - Apply security guardrails to MCP calls
- [Using MCP](./mcp_usage.md) - How to use MCP with LiteLLM
:::
## How It Works
LiteLLM supports managing permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. When a MCP client attempts to list tools, LiteLLM will only return the tools the entity has permissions to access.
When Creating a Key, Team, or Organization, you can select the allowed MCP Servers that the entity has access to.
<Image
img={require('../img/mcp_key.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
## Set Allowed Tools for a Key, Team, or Organization
Control which tools different teams can access from the same MCP server. For example, give your Engineering team access to `list_repositories`, `create_issue`, and `search_code`, while Sales only gets `search_code` and `close_issue`.
This video shows how to set allowed tools for a Key, Team, or Organization.
<iframe width="840" height="500" src="https://www.loom.com/embed/7464d444c3324078892367272fe50745" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>

121
docs/my-website/docs/mcp_cost.md Normal file
View file

@ -0,0 +1,121 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# MCP Cost Tracking
LiteLLM provides two ways to track costs for MCP tool calls:
| Method | When to Use | What It Does |
|--------|-------------|--------------|
| **Config-based Cost Tracking** | Simple cost tracking with fixed costs per tool/server | Automatically tracks costs based on configuration |
| **Custom Post-MCP Hook** | Dynamic cost tracking with custom logic | Allows custom cost calculations and response modifications |
### Config-based Cost Tracking
Configure fixed costs for MCP servers directly in your config.yaml:
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-xxxxxxx
mcp_servers:
zapier_server:
url: "https://actions.zapier.com/mcp/sk-xxxxx/sse"
mcp_info:
mcp_server_cost_info:
# Default cost for all tools in this server
default_cost_per_query: 0.01
# Custom cost for specific tools
tool_name_to_cost_per_query:
send_email: 0.05
create_document: 0.03
expensive_api_server:
url: "https://api.expensive-service.com/mcp"
mcp_info:
mcp_server_cost_info:
default_cost_per_query: 1.50
```
### Custom Post-MCP Hook
Use this when you need dynamic cost calculation or want to modify the MCP response before it's returned to the user.
#### 1. Create a custom MCP hook file
```python title="custom_mcp_hook.py" showLineNumbers
from typing import Optional
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.mcp import MCPPostCallResponseObject
class CustomMCPCostTracker(CustomLogger):
"""
Custom handler for MCP cost tracking and response modification
"""
async def async_post_mcp_tool_call_hook(
self,
kwargs,
response_obj: MCPPostCallResponseObject,
start_time,
end_time
) -> Optional[MCPPostCallResponseObject]:
"""
Called after each MCP tool call.
Modify costs and response before returning to user.
"""
# Extract tool information from kwargs
tool_name = kwargs.get("name", "")
server_name = kwargs.get("server_name", "")
# Calculate custom cost based on your logic
custom_cost = 42.00
# Set the response cost
response_obj.hidden_params.response_cost = custom_cost
return response_obj
# Create instance for LiteLLM to use
custom_mcp_cost_tracker = CustomMCPCostTracker()
```
#### 2. Configure in config.yaml
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-xxxxxxx
# Add your custom MCP hook
callbacks:
- custom_mcp_hook.custom_mcp_cost_tracker
mcp_servers:
zapier_server:
url: "https://actions.zapier.com/mcp/sk-xxxxx/sse"
```
#### 3. Start the proxy
```shell
$ litellm --config /path/to/config.yaml
```
When MCP tools are called, your custom hook will:
1. Calculate costs based on your custom logic
2. Modify the response if needed
3. Track costs in LiteLLM's logging system

88
docs/my-website/docs/mcp_guardrail.md Normal file
View file

@ -0,0 +1,88 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# MCP Guardrails
LiteLLM supports applying guardrails to MCP tool calls to ensure security and compliance. You can configure guardrails to run before or during MCP calls to validate inputs and block or mask sensitive information.
### Supported MCP Guardrail Modes
MCP guardrails support the following modes:
- `pre_mcp_call`: Run **before** MCP call, on **input**. Use this mode when you want to apply validation/masking/blocking for MCP requests
- `during_mcp_call`: Run **during** MCP call execution. Use this mode for real-time monitoring and intervention
### Configuration Examples
Configure guardrails to run before MCP tool calls to validate and sanitize inputs:
```yaml title="config.yaml" showLineNumbers
guardrails:
- guardrail_name: "mcp-input-validation"
litellm_params:
guardrail: presidio # or other supported guardrails
mode: "pre_mcp_call" # or during_mcp_call
pii_entities_config:
CREDIT_CARD: "BLOCK" # Will block requests containing credit card numbers
EMAIL_ADDRESS: "MASK" # Will mask email addresses
PHONE_NUMBER: "MASK" # Will mask phone numbers
default_on: true
```
### Usage Examples
#### Testing Pre-MCP Call Guardrails
Test your MCP guardrails with a request that includes sensitive information:
```bash title="Test MCP Guardrail" showLineNumbers
curl http://localhost:4000/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{"role": "user", "content": "My credit card is 4111-1111-1111-1111 and my email is john@example.com"}
],
"guardrails": ["mcp-input-validation"]
}'
```
The request will be processed as follows:
1. Credit card number will be blocked (request rejected)
2. Email address will be masked (e.g., replaced with `<EMAIL_ADDRESS>`)
#### Using with MCP Tools
When using MCP tools, guardrails will be applied to the tool inputs:
```python title="Python Example with MCP Guardrails" showLineNumbers
import openai
client = openai.OpenAI(
api_key="your-api-key",
base_url="http://localhost:4000"
)
# This request will trigger MCP guardrails
response = client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "user", "content": "Send an email to 555-123-4567 with my SSN 123-45-6789"}
],
tools=[{"type": "mcp", "server_label": "litellm", "server_url": "litellm_proxy"}],
guardrails=["mcp-input-validation"]
)
```
### Supported Guardrail Providers
MCP guardrails work with all LiteLLM-supported guardrail providers:
- **Presidio**: PII detection and masking
- **Bedrock**: AWS Bedrock guardrails
- **Lakera**: Content moderation
- **Aporia**: Custom guardrails
- **Custom**: Your own guardrail implementations

209
docs/my-website/docs/mcp_usage.md Normal file
View file

@ -0,0 +1,209 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# Using your MCP
This document covers how to use LiteLLM as an MCP Gateway. You can see how to use it with Responses API, Cursor IDE, and OpenAI SDK.
### Use on LiteLLM UI
Follow this walkthrough to use your MCP on LiteLLM UI
<iframe width="840" height="500" src="https://www.loom.com/embed/57e0763267254bc79dbe6658d0b8758c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
### Use with Responses API
Replace `http://localhost:4000` with your LiteLLM Proxy base URL.
Demo Video Using Responses API with LiteLLM Proxy: [Demo video here](https://www.loom.com/share/34587e618c5c47c0b0d67b4e4d02718f?sid=2caf3d45-ead4-4490-bcc1-8d6dd6041c02)
<Tabs>
<TabItem value="curl" label="cURL">
```bash title="cURL Example" showLineNumbers
curl --location 'http://localhost:4000/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer sk-1234" \
--data '{
"model": "gpt-5",
"input": [
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
}
],
"tools": [
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never"
}
],
"stream": true,
"tool_choice": "required"
}'
```
</TabItem>
<TabItem value="python" label="Python SDK">
```python title="Python SDK Example" showLineNumbers
"""
Use LiteLLM Proxy MCP Gateway to call MCP tools.
When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
"""
import openai
client = openai.OpenAI(
api_key="sk-1234", # paste your litellm proxy api key here
base_url="http://localhost:4000" # paste your litellm proxy base url here
)
print("Making API request to Responses API with MCP tools")
response = client.responses.create(
model="gpt-5",
input=[
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
}
],
tools=[
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never"
}
],
stream=True,
tool_choice="required"
)
for chunk in response:
print("response chunk: ", chunk)
```
</TabItem>
</Tabs>
#### Specifying MCP Tools
You can specify which MCP tools are available by using the `allowed_tools` parameter. This allows you to restrict access to specific tools within an MCP server.
To get the list of allowed tools when using LiteLLM MCP Gateway, you can naigate to the LiteLLM UI on MCP Servers > MCP Tools > Click the Tool > Copy Tool Name.
<Tabs>
<TabItem value="curl" label="cURL">
```bash title="cURL Example with allowed_tools" showLineNumbers
curl --location 'http://localhost:4000/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer sk-1234" \
--data '{
"model": "gpt-5",
"input": [
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
}
],
"tools": [
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy/mcp",
"require_approval": "never",
"allowed_tools": ["GitMCP-fetch_litellm_documentation"]
}
],
"stream": true,
"tool_choice": "required"
}'
```
</TabItem>
<TabItem value="python" label="Python SDK">
```python title="Python SDK Example with allowed_tools" showLineNumbers
import openai
client = openai.OpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
response = client.responses.create(
model="gpt-5",
input=[
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
}
],
tools=[
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy/mcp",
"require_approval": "never",
"allowed_tools": ["GitMCP-fetch_litellm_documentation"]
}
],
stream=True,
tool_choice="required"
)
print(response)
```
</TabItem>
</Tabs>
### Use with Cursor IDE
Use tools directly from Cursor IDE with LiteLLM MCP:
**Setup Instructions:**
1. **Open Cursor Settings**: Use `⇧+⌘+J` (Mac) or `Ctrl+Shift+J` (Windows/Linux)
2. **Navigate to MCP Tools**: Go to the "MCP Tools" tab and click "New MCP Server"
3. **Add Configuration**: Copy and paste the JSON configuration below, then save with `Cmd+S` or `Ctrl+S`
```json title="Basic Cursor MCP Configuration" showLineNumbers
{
"mcpServers": {
"LiteLLM": {
"url": "litellm_proxy",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY"
}
}
}
}
```
#### How it works when server_url="litellm_proxy"
When server_url="litellm_proxy", LiteLLM bridges non-MCP providers to your MCP tools.
- Tool Discovery: LiteLLM fetches MCP tools and converts them to OpenAI-compatible definitions
- LLM Call: Tools are sent to the LLM with your input; LLM selects which tools to call
- Tool Execution: LiteLLM automatically parses arguments, routes calls to MCP servers, executes tools, and retrieves results
- Response Integration: Tool results are sent back to LLM for final response generation
- Output: Complete response combining LLM reasoning with tool execution results
This enables MCP tool usage with any LiteLLM-supported provider, regardless of native MCP support.
#### Auto-execution for require_approval: "never"
Setting require_approval: "never" triggers automatic tool execution, returning the final response in a single API call without additional user interaction.

45
docs/my-website/docs/observability/posthog_integration.md
View file

@ -55,6 +55,26 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
}'
```
### Team-Based Logging
Configure different PostHog credentials per team using the team callback settings:
```bash
curl -X POST 'http://localhost:4000/team/{team_id}/callback' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"callback_name": "posthog",
"callback_type": "success",
"callback_vars": {
"posthog_api_key": "ph_team_specific_key",
"posthog_api_url": "https://custom.posthog.com"
}
}'
```
Now all requests from that team will be logged to their specific PostHog project.
## Usage with LiteLLM Python SDK
### Quick Start
@ -142,6 +162,31 @@ response = client.chat.completions.create(
)
```
#### Per-Request Credentials
You can override PostHog credentials on a per-request basis:
```python
import litellm
litellm.success_callback = ["posthog"]
# Use custom PostHog credentials for this specific request
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[
{"role": "user", "content": "Hello world"}
],
posthog_api_key="ph_custom_project_key",
posthog_api_url="https://custom.posthog.com"
)
```
This is useful when you need to:
- Log different teams/projects to separate PostHog instances
- Use different PostHog projects for staging vs production
- Route logs based on customer or tenant
#### Disable Logging for Specific Calls
Use the `no-log` flag to prevent logging for specific calls:

257
docs/my-website/docs/ocr.md Normal file
View file

@ -0,0 +1,257 @@
# /ocr
:::tip
LiteLLM follows the [Mistral API request/response for the OCR API](https://docs.mistral.ai/capabilities/vision/#optical-character-recognition-ocr)
:::
## **LiteLLM Python SDK Usage**
### Quick Start
```python
from litellm import ocr
import os
os.environ["MISTRAL_API_KEY"] = "sk-.."
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "document_url",
"document_url": "https://arxiv.org/pdf/2201.04234"
}
)
# Access extracted text
for page in response.pages:
print(f"Page {page.index}:")
print(page.markdown)
```
### Async Usage
```python
from litellm import aocr
import os, asyncio
os.environ["MISTRAL_API_KEY"] = "sk-.."
async def test_async_ocr():
response = await aocr(
model="mistral/mistral-ocr-latest",
document={
"type": "document_url",
"document_url": "https://arxiv.org/pdf/2201.04234"
}
)
# Access extracted text
for page in response.pages:
print(f"Page {page.index}:")
print(page.markdown)
asyncio.run(test_async_ocr())
```
### Using Base64 Encoded Documents
```python
import base64
from litellm import ocr
# Encode PDF to base64
with open("document.pdf", "rb") as f:
base64_pdf = base64.b64encode(f.read()).decode('utf-8')
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "document_url",
"document_url": f"data:application/pdf;base64,{base64_pdf}"
}
)
```
### Optional Parameters
```python
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "document_url",
"document_url": "https://example.com/doc.pdf"
},
# Optional Mistral parameters
pages=[0, 1, 2], # Only process specific pages
include_image_base64=True, # Include extracted images
image_limit=10, # Max images to return
image_min_size=100 # Min image size to include
)
```
## **LiteLLM Proxy Usage**
LiteLLM provides a Mistral API compatible `/ocr` endpoint for OCR calls.
**Setup**
Add this to your litellm proxy config.yaml
```yaml
model_list:
- model_name: mistral-ocr
litellm_params:
model: mistral/mistral-ocr-latest
api_key: os.environ/MISTRAL_API_KEY
```
Start litellm
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
Test request
```bash
curl http://0.0.0.0:4000/v1/ocr \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "mistral-ocr",
"document": {
"type": "document_url",
"document_url": "https://arxiv.org/pdf/2201.04234"
}
}'
```
## **Request/Response Format**
:::info
LiteLLM follows the **Mistral OCR API specification**.
See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilities/vision/#optical-character-recognition-ocr) for complete details.
:::
### Example Request
```python
{
"model": "mistral/mistral-ocr-latest",
"document": {
"type": "document_url",
"document_url": "https://arxiv.org/pdf/2201.04234"
},
"pages": [0, 1, 2], # Optional: specific pages to process
"include_image_base64": True, # Optional: include extracted images
"image_limit": 10, # Optional: max images to return
"image_min_size": 100 # Optional: min image size in pixels
}
```
### Request Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) |
| `document` | object | Yes | Document to process. Must contain `type` and URL field |
| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images |
| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) |
| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) |
| `pages` | array | No | List of specific page indices to process (0-indexed) |
| `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings |
| `image_limit` | integer | No | Maximum number of images to return |
| `image_min_size` | integer | No | Minimum size (in pixels) for images to include |
#### Document Format Examples
**For PDFs and documents:**
```json
{
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
```
**For images:**
```json
{
"type": "image_url",
"image_url": "https://example.com/image.png"
}
```
**For base64-encoded content:**
```json
{
"type": "document_url",
"document_url": "data:application/pdf;base64,JVBERi0xLjQKJ..."
}
```
### Response Format
The response follows Mistral's OCR format with the following structure:
```json
{
"pages": [
{
"index": 0,
"markdown": "# Document Title\n\nExtracted text content...",
"dimensions": {
"dpi": 200,
"height": 2200,
"width": 1700
},
"images": [
{
"image_base64": "base64string...",
"bbox": {
"x": 100,
"y": 200,
"width": 300,
"height": 400
}
}
]
}
],
"model": "mistral-ocr-2505-completion",
"usage_info": {
"pages_processed": 29,
"doc_size_bytes": 3002783
},
"document_annotation": null,
"object": "ocr"
}
```
#### Response Fields
| Field | Type | Description |
|-------|------|-------------|
| `pages` | array | List of processed pages with extracted content |
| `pages[].index` | integer | Page number (0-indexed) |
| `pages[].markdown` | string | Extracted text in Markdown format |
| `pages[].dimensions` | object | Page dimensions (dpi, height, width in pixels) |
| `pages[].images` | array | Extracted images from the page (if `include_image_base64=true`) |
| `model` | string | The model used for OCR processing |
| `usage_info` | object | Processing statistics (pages processed, document size) |
| `document_annotation` | object | Optional document-level annotations |
| `object` | string | Always `"ocr"` for OCR responses |
## **Supported Providers**
| Provider | Link to Usage |
|-------------|--------------------|
| Mistral AI | [Usage](#quick-start) |

177
docs/my-website/docs/providers/bedrock_embedding.md
View file

@ -8,6 +8,182 @@
| Cohere | `bedrock/cohere.*` | [Cohere Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-embed.html) |
| TwelveLabs | `bedrock/us.twelvelabs.*` | [TwelveLabs](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-twelvelabs.html) |
## Async Invoke Support
LiteLLM supports AWS Bedrock's async-invoke feature for embedding models that require asynchronous processing, particularly useful for large media files (video, audio) or when you need to process embeddings in the background.
### Supported Models
| Provider | Async Invoke Route | Use Case |
|----------|-------------------|----------|
| TwelveLabs Marengo | `bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0` | Video, audio, image, and text embeddings |
### Required Parameters
When using async-invoke, you must provide:
| Parameter | Description | Required |
|-----------|-------------|----------|
| `output_s3_uri` | S3 URI where the embedding results will be stored | ✅ Yes |
| `input_type` | Type of input: `"text"`, `"image"`, `"video"`, or `"audio"` | ✅ Yes |
| `aws_region_name` | AWS region for the request | ✅ Yes |
### Usage
#### Basic Async Invoke
```python
from litellm import embedding
# Text embedding with async-invoke
response = embedding(
model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
input=["Hello world from LiteLLM async invoke!"],
aws_region_name="us-east-1",
input_type="text",
output_s3_uri="s3://your-bucket/async-invoke-output/"
)
print(f"Job submitted! Invocation ARN: {response._hidden_params._invocation_arn}")
```
#### Video/Audio Embedding
```python
# Video embedding (requires async-invoke)
response = embedding(
model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
input=["s3://your-bucket/video.mp4"], # S3 URL for video
aws_region_name="us-east-1",
input_type="video",
output_s3_uri="s3://your-bucket/async-invoke-output/"
)
print(f"Video embedding job submitted! ARN: {response._hidden_params._invocation_arn}")
```
#### Image Embedding with Base64
```python
import base64
# Load and encode image
with open("image.jpg", "rb") as img_file:
img_data = base64.b64encode(img_file.read()).decode('utf-8')
img_base64 = f"data:image/jpeg;base64,{img_data}"
response = embedding(
model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
input=[img_base64],
aws_region_name="us-east-1",
input_type="image",
output_s3_uri="s3://your-bucket/async-invoke-output/"
)
```
### Retrieving Job Information
#### Getting Job ID and Invocation ARN
The async-invoke response includes the invocation ARN in the hidden parameters:
```python
response = embedding(
model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
input=["Hello world"],
aws_region_name="us-east-1",
input_type="text",
output_s3_uri="s3://your-bucket/async-invoke-output/"
)
# Access invocation ARN
invocation_arn = response._hidden_params._invocation_arn
print(f"Invocation ARN: {invocation_arn}")
# Extract job ID from ARN (last part after the last slash)
job_id = invocation_arn.split("/")[-1]
print(f"Job ID: {job_id}")
```
#### Checking Job Status
Use LiteLLM's `retrieve_batch` function to check if your job is still processing:
```python
from litellm import retrieve_batch
def check_async_job_status(invocation_arn, aws_region_name="us-east-1"):
"""Check the status of an async invoke job using LiteLLM batch API"""
try:
response = retrieve_batch(
batch_id=invocation_arn,
custom_llm_provider="bedrock",
aws_region_name=aws_region_name
)
return response
except Exception as e:
print(f"Error checking job status: {e}")
return None
# Check status
status = check_async_job_status(invocation_arn, "us-east-1")
if status:
print(f"Job Status: {status.status}")
print(f"Output Location: {status.output_file_id}")
```
**Note:** The actual embedding results are stored in S3. The `output_file_id` from the batch status can be used to locate the results file in your S3 bucket.
### Error Handling
#### Common Errors
| Error | Cause | Solution |
|-------|-------|----------|
| `ValueError: output_s3_uri cannot be empty` | Missing S3 output URI | Provide a valid S3 URI |
| `ValueError: Input type 'video' requires async_invoke route` | Using video/audio without async-invoke | Use `bedrock/async_invoke/` model prefix |
| `ValueError: input_type is required` | Missing input type parameter | Specify `input_type` parameter |
#### Example Error Handling
```python
try:
response = embedding(
model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
input=["Hello world"],
aws_region_name="us-east-1",
input_type="text",
output_s3_uri="s3://your-bucket/output/" # Required for async-invoke
)
print("Job submitted successfully!")
except ValueError as e:
if "output_s3_uri cannot be empty" in str(e):
print("Error: Please provide a valid S3 output URI")
elif "requires async_invoke route" in str(e):
print("Error: Use async_invoke model for video/audio inputs")
else:
print(f"Error: {e}")
except Exception as e:
print(f"Unexpected error: {e}")
```
### Best Practices
1. **Use async-invoke for large files**: Video and audio files are better processed asynchronously
2. **Use LiteLLM batch API**: Use `retrieve_batch()` instead of direct Bedrock API calls for status checking
3. **Monitor job status**: Check job status periodically using the batch API to know when results are ready
4. **Handle errors gracefully**: Implement proper error handling for network issues and job failures
5. **Set appropriate timeouts**: Consider the processing time for large files
6. **Use S3 for large inputs**: For video/audio, use S3 URLs instead of base64 encoding
### Limitations
- Async-invoke is currently only supported for TwelveLabs Marengo models
- Results are stored in S3 and must be retrieved separately using the output file ID
- Job status checking requires using LiteLLM's `retrieve_batch()` function
- No built-in polling mechanism in LiteLLM (must implement your own status checking loop)
### API keys
This can be set as env variables or passed as **params to litellm.embedding()**
```python
@ -89,6 +265,7 @@ print(response)
| TwelveLabs Marengo Embed 2.7 | `embedding(model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0", input=input)` | Supports multimodal input (text, video, audio, image) |
| Cohere Embeddings - English | `embedding(model="bedrock/cohere.embed-english-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
| Cohere Embeddings - Multilingual | `embedding(model="bedrock/cohere.embed-multilingual-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
| Cohere Embed v4 | `embedding(model="bedrock/cohere.embed-v4:0", input=input)` | Supports text and image input, configurable dimensions (256, 512, 1024, 1536), 128k context length |
### Advanced - [Drop Unsupported Params](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)

4
docs/my-website/docs/providers/cometapi.md
View file

@ -1,6 +1,10 @@
# CometAPI
LiteLLM supports all AI models from [CometAPI](https://www.cometapi.com/). CometAPI provides access to 500+ AI models through a unified API interface, including cutting-edge models like GPT-5, Claude Opus 4.1, and various other state-of-the-art language models.
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_CometAPI.ipynb">
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
</a>
## Authentication
To use CometAPI models, you need to obtain an API key from [CometAPI Token Console](https://api.cometapi.com/console/token). CometAPI offers free tokens for new users - you can get your free API key instantly by registering.

4
docs/my-website/docs/providers/nvidia_nim.md
View file

@ -15,8 +15,8 @@ https://docs.api.nvidia.com/nim/reference/
| Description | Nvidia NIM is a platform that provides a simple API for deploying and using AI models. LiteLLM supports all models from [Nvidia NIM](https://developer.nvidia.com/nim/) |
| Provider Route on LiteLLM | `nvidia_nim/` |
| Provider Doc | [Nvidia NIM Docs ↗](https://developer.nvidia.com/nim/) |
| API Endpoint for Provider | https://integrate.api.nvidia.com/v1/ |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/responses`, `/embeddings` |
| API Endpoint for Provider | https://integrate.api.nvidia.com/v1/ (chat/embeddings), https://ai.api.nvidia.com/v1/ (rerank) |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/responses`, `/embeddings`, `/rerank` |
## API Key
```python

261
docs/my-website/docs/providers/nvidia_nim_rerank.md Normal file
View file

@ -0,0 +1,261 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Nvidia NIM - Rerank
Use Nvidia NIM Rerank models through LiteLLM.
| Property | Details |
|----------|---------|
| Description | Nvidia NIM provides high-performance reranking models for semantic search and retrieval-augmented generation (RAG) |
| Provider Doc | [Nvidia NIM Rerank API ↗](https://docs.api.nvidia.com/nim/reference/nvidia-llama-3_2-nv-rerankqa-1b-v2-infer) |
| Supported Endpoint | `/rerank` |
## Overview
Nvidia NIM rerank models help you:
- Reorder search results by relevance to a query
- Improve RAG (Retrieval-Augmented Generation) accuracy
- Filter and rank large document sets efficiently
**Supported Models:**
- All Nvidia NIM rerank models on their platform
:::tip
See the full list of LiteLLM supported Nvidia NIM rerank models on [Nvidia NIM](https://models.litellm.ai)
:::
## Usage
### LiteLLM Python SDK
<Tabs>
<TabItem value="llama-1b" label="LLaMa 1B Model">
```python
import litellm
import os
os.environ['NVIDIA_NIM_API_KEY'] = "nvapi-..."
response = litellm.rerank(
model="nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2",
query="What is the GPU memory bandwidth of H100 SXM?",
documents=[
"The Hopper GPU is paired with the Grace CPU using NVIDIA's ultra-fast chip-to-chip interconnect, delivering 900GB/s of bandwidth.",
"A100 provides up to 20X higher performance over the prior generation.",
"Accelerated servers with H100 deliver 3 terabytes per second (TB/s) of memory bandwidth per GPU."
],
top_n=3,
)
print(response)
```
</TabItem>
<TabItem value="mistral-4b" label="Mistral 4B Model">
```python
import litellm
import os
os.environ['NVIDIA_NIM_API_KEY'] = "nvapi-..."
response = litellm.rerank(
model="nvidia_nim/nvidia/nv-rerankqa-mistral-4b-v3",
query="What is the GPU memory bandwidth of H100 SXM?",
documents=[
"The Hopper GPU is paired with the Grace CPU using NVIDIA's ultra-fast chip-to-chip interconnect, delivering 900GB/s of bandwidth.",
"A100 provides up to 20X higher performance over the prior generation.",
"Accelerated servers with H100 deliver 3 terabytes per second (TB/s) of memory bandwidth per GPU."
],
top_n=3,
)
print(response)
```
</TabItem>
</Tabs>
**Response:**
```json
{
"results": [
{
"index": 2,
"relevance_score": 6.828125,
"document": {
"text": "Accelerated servers with H100 deliver 3 terabytes per second (TB/s) of memory bandwidth per GPU."
}
},
{
"index": 0,
"relevance_score": -1.564453125,
"document": {
"text": "The Hopper GPU is paired with the Grace CPU using NVIDIA's ultra-fast chip-to-chip interconnect, delivering 900GB/s of bandwidth."
}
}
]
}
```
## Usage with LiteLLM Proxy
### 1. Setup Config
Add Nvidia NIM rerank models to your proxy configuration:
```yaml
model_list:
- model_name: nvidia-rerank
litellm_params:
model: nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2
api_key: os.environ/NVIDIA_NIM_API_KEY
```
### 2. Start Proxy
```bash
litellm --config /path/to/config.yaml
```
### 3. Make Rerank Requests
```bash
curl -X POST http://0.0.0.0:4000/rerank \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "nvidia-rerank",
"query": "What is the GPU memory bandwidth of H100?",
"documents": [
"H100 delivers 3TB/s memory bandwidth",
"A100 has 2TB/s memory bandwidth",
"V100 offers 900GB/s memory bandwidth"
],
"top_n": 2
}'
```
## API Parameters
### Required Parameters
| Parameter | Type | Description |
|-----------|------|-------------|
| `model` | string | The Nvidia NIM rerank model name with `nvidia_nim/` prefix |
| `query` | string | The search query to rank documents against |
| `documents` | array | List of documents to rank (1-1000 documents) |
### Optional Parameters
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `top_n` | integer | All documents | Number of top-ranked documents to return |
### Nvidia-Specific Parameters
**`truncate`**: Controls how text is truncated if it exceeds the model's context window
- `"NONE"`: No truncation (request may fail if too long)
- `"END"`: Truncate from the end of the text
```python
response = litellm.rerank(
model="nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2",
query="GPU performance",
documents=["High performance computing", "Fast GPU processing"],
top_n=2,
truncate="END", # Nvidia-specific parameter
)
```
## Authentication
Set your Nvidia NIM API key:
<Tabs>
<TabItem value="env" label="Environment Variable">
```bash
export NVIDIA_NIM_API_KEY="nvapi-..."
```
</TabItem>
<TabItem value="python" label="Python">
```python
import os
os.environ['NVIDIA_NIM_API_KEY'] = "nvapi-..."
# Or pass directly
response = litellm.rerank(
model="nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2",
query="test",
documents=["doc1"],
api_key="nvapi-...",
)
```
</TabItem>
</Tabs>
## API Endpoint
The rerank endpoint uses a different base URL than chat/embeddings:
- **Chat/Embeddings:** `https://integrate.api.nvidia.com/v1/`
- **Rerank:** `https://ai.api.nvidia.com/v1/`
LiteLLM automatically uses the correct endpoint for rerank requests.
### Custom API Base URL
You can override the default base URL in several ways:
**Option 1: Environment Variable**
```bash
export NVIDIA_NIM_API_BASE="https://your-custom-endpoint.com"
```
**Option 2: Pass as parameter**
```python
response = litellm.rerank(
model="nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2",
query="test",
documents=["doc1"],
api_base="https://your-custom-endpoint.com",
)
```
**Option 3: Full URL (including model path)**
If you have the complete endpoint URL, you can pass it directly:
```python
response = litellm.rerank(
model="nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2",
query="test",
documents=["doc1"],
api_base="https://your-custom-endpoint.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking",
)
```
LiteLLM will detect the full URL (by checking for `/retrieval/` in the path) and use it as-is.
### How do I get an API key?
Get your Nvidia NIM API key from [Nvidia's website](https://developer.nvidia.com/nim/).
## Related Documentation
- [Nvidia NIM - Main Documentation](./nvidia_nim)
- [Nvidia NIM Chat Completions](./nvidia_nim#sample-usage)
- [LiteLLM Rerank Endpoint](../rerank)
- [Nvidia NIM Official Docs ↗](https://docs.api.nvidia.com/nim/reference/)

32
docs/my-website/docs/providers/oci.md
View file

@ -6,18 +6,27 @@ LiteLLM supports the following models for OCI on-demand GenAI API.
Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm) to see if the model is available for your region.
## Supported Models
### Meta Llama Models
- `meta.llama-4-maverick-17b-128e-instruct-fp8`
- `meta.llama-4-scout-17b-16e-instruct`
- `meta.llama-3.3-70b-instruct`
- `meta.llama-3.2-90b-vision-instruct`
- `meta.llama-3.1-405b-instruct`
### xAI Grok Models
- `xai.grok-4`
- `xai.grok-3`
- `xai.grok-3-fast`
- `xai.grok-3-mini`
- `xai.grok-3-mini-fast`
### Cohere Models
- `cohere.command-latest`
- `cohere.command-a-03-2025`
- `cohere.command-plus-latest`
## Authentication
LiteLLM uses OCI signing key authentication. Follow the [official Oracle tutorial](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm) to create a signing key and obtain the following parameters:
@ -44,6 +53,7 @@ response = completion(
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
oci_serving_mode="ON_DEMAND", # Optional, default is "ON_DEMAND". Other option is "DEDICATED"
# Provide either the private key string OR the path to the key file:
# Option 1: pass the private key as a string
oci_key=<string_with_content_of_oci_key>,
@ -71,6 +81,7 @@ response = completion(
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
oci_serving_mode="ON_DEMAND", # Optional, default is "ON_DEMAND". Other option is "DEDICATED"
# Provide either the private key string OR the path to the key file:
# Option 1: pass the private key as a string
oci_key=<string_with_content_of_oci_key>,
@ -81,3 +92,24 @@ response = completion(
for chunk in response:
print(chunk["choices"][0]["delta"]["content"]) # same as openai format
```
## Usage Examples by Model Type
### Using Cohere Models
```python
from litellm import completion
messages = [{"role": "user", "content": "Explain quantum computing"}]
response = completion(
model="oci/cohere.command-latest",
messages=messages,
oci_region="us-chicago-1",
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
oci_key=<string_with_content_of_oci_key>,
oci_compartment_id=<oci_compartment_id>,
)
print(response)
```

87
docs/my-website/docs/providers/openai.md
View file

@ -171,6 +171,7 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL
| gpt-5-2025-08-07 | `response = completion(model="gpt-5-2025-08-07", messages=messages)` |
| gpt-5-mini-2025-08-07 | `response = completion(model="gpt-5-mini-2025-08-07", messages=messages)` |
| gpt-5-nano-2025-08-07 | `response = completion(model="gpt-5-nano-2025-08-07", messages=messages)` |
| gpt-5-pro | `response = completion(model="gpt-5-pro", messages=messages)` |
| gpt-4.1 | `response = completion(model="gpt-4.1", messages=messages)` |
| gpt-4.1-mini | `response = completion(model="gpt-4.1-mini", messages=messages)` |
| gpt-4.1-nano | `response = completion(model="gpt-4.1-nano", messages=messages)` |
@ -338,6 +339,72 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
| fine tuned `gpt-3.5-turbo-1106` | `response = completion(model="ft:gpt-3.5-turbo-1106", messages=messages)` |
| fine tuned `gpt-3.5-turbo-0613` | `response = completion(model="ft:gpt-3.5-turbo-0613", messages=messages)` |
## Getting Reasoning Content in `/chat/completions`
GPT-5 models return reasoning content when called via the Responses API. You can call these models via the `/chat/completions` endpoint by using the `openai/responses/` prefix.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.completion(
model="openai/responses/gpt-5-mini", # tells litellm to call the model via the Responses API
messages=[{"role": "user", "content": "What is the capital of France?"}],
reasoning_effort="low",
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "openai/responses/gpt-5-mini",
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"reasoning_effort": "low"
}'
```
</TabItem>
</Tabs>
Expected Response:
```json
{
"id": "chatcmpl-6382a222-43c9-40c4-856b-22e105d88075",
"created": 1760146746,
"model": "gpt-5-mini",
"object": "chat.completion",
"system_fingerprint": null,
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Paris",
"role": "assistant",
"tool_calls": null,
"function_call": null,
"reasoning_content": "**Identifying the capital**\n\nThe user wants me to think of the capital of France and write it down. That's pretty straightforward: it's Paris. There aren't any safety issues to consider here. I think it would be best to keep it concise, so maybe just \"Paris\" would suffice. I feel confident that I should just stick to that without adding anything else. So, let's write it down!",
"provider_specific_fields": null
}
}
],
"usage": {
"completion_tokens": 7,
"prompt_tokens": 18,
"total_tokens": 25,
"completion_tokens_details": null,
"prompt_tokens_details": {
"audio_tokens": null,
"cached_tokens": 0,
"text_tokens": null,
"image_tokens": null
}
}
}
```
## OpenAI Chat Completion to Responses API Bridge
@ -749,4 +816,24 @@ In your logs you should see the forwarded org id
```bash
LiteLLM:DEBUG: utils.py:255 - Request to litellm:
LiteLLM:DEBUG: utils.py:255 - litellm.acompletion(... organization='my-special-org',)
```
## GPT-5 Pro Special Notes
GPT-5 Pro is OpenAI's most advanced reasoning model with unique characteristics:
- **Responses API Only**: GPT-5 Pro is only available through the `/v1/responses` endpoint
- **No Streaming**: Does not support streaming responses
- **High Reasoning**: Designed for complex reasoning tasks with highest effort reasoning
- **Context Window**: 400,000 tokens input, 272,000 tokens output
- **Pricing**: $15.00 input / $120.00 output per 1M tokens (Standard), $7.50 input / $60.00 output (Batch)
- **Tools**: Supports Web Search, File Search, Image Generation, MCP (but not Code Interpreter or Computer Use)
- **Modalities**: Text and Image input, Text output only
```python
# GPT-5 Pro usage example
response = completion(
model="gpt-5-pro",
messages=[{"role": "user", "content": "Solve this complex reasoning problem..."}]
)
```

50
docs/my-website/docs/providers/openai/responses_api.md
View file

@ -37,6 +37,29 @@ for event in response:
print(event)
```
#### Image Generation with Streaming
```python showLineNumbers title="OpenAI Streaming Image Generation"
import litellm
import base64
# Streaming image generation with partial images
stream = litellm.responses(
model="gpt-4.1", # Use an actual image generation model
input="Generate a gorgeous image of a river made of white owl feathers",
stream=True,
tools=[{"type": "image_generation", "partial_images": 2}],
)
for event in stream:
if event.type == "response.image_generation_call.partial_image":
idx = event.partial_image_index
image_base64 = event.partial_image_b64
image_bytes = base64.b64decode(image_base64)
with open(f"river{idx}.png", "wb") as f:
f.write(image_bytes)
```
#### GET a Response
```python showLineNumbers title="Get Response by ID"
import litellm
@ -150,6 +173,33 @@ for event in response:
print(event)
```
#### Image Generation with Streaming
```python showLineNumbers title="OpenAI Proxy Streaming Image Generation"
from openai import OpenAI
import base64
# Initialize client with your proxy URL
client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
stream = client.responses.create(
model="gpt-4.1",
input="Draw a gorgeous image of a river made of white owl feathers, snaking its way through a serene winter landscape",
stream=True,
tools=[{"type": "image_generation", "partial_images": 2}],
)
for event in stream:
print(f"event: {event}")
if event.type == "response.image_generation_call.partial_image":
idx = event.partial_image_index
image_base64 = event.partial_image_b64
image_bytes = base64.b64decode(image_base64)
with open(f"river{idx}.png", "wb") as f:
f.write(image_bytes)
```
#### GET a Response
```python showLineNumbers title="Get Response by ID with OpenAI SDK"
from openai import OpenAI

316
docs/my-website/docs/providers/vertex.md
View file

@ -191,7 +191,7 @@ print(json.loads(completion.choices[0].message.content))
model_list:
- model_name: gemini-2.5-pro
litellm_params:
model: vertex_ai/gemini-1.5-pro
model: vertex_ai/gemini-2.5-pro
vertex_project: "project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env
@ -277,7 +277,7 @@ except JSONSchemaValidationError as e:
model_list:
- model_name: gemini-2.5-pro
litellm_params:
model: vertex_ai/gemini-1.5-pro
model: vertex_ai/gemini-2.5-pro
vertex_project: "project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env
@ -621,6 +621,163 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
#### **Google Maps**
Use Google Maps to provide location-based context to your Gemini models.
[**Relevant Vertex AI Docs**](https://ai.google.dev/gemini-api/docs/grounding#google-maps)
<Tabs>
<TabItem value="sdk" label="SDK">
**Basic Usage - Enable Widget Only**
```python showLineNumbers
from litellm import completion
## SETUP ENVIRONMENT
# !gcloud auth application-default login - run this to add vertex credentials to your env
tools = [{"googleMaps": {"enableWidget": "ENABLE_WIDGET"}}] # 👈 ADD GOOGLE MAPS
resp = litellm.completion(
model="vertex_ai/gemini-2.0-flash",
messages=[{"role": "user", "content": "What restaurants are nearby?"}],
tools=tools,
)
print(resp)
```
**With Location Data**
You can specify a location to ground the model's responses with location-specific information:
```python showLineNumbers
from litellm import completion
## SETUP ENVIRONMENT
# !gcloud auth application-default login - run this to add vertex credentials to your env
tools = [{
"googleMaps": {
"enableWidget": "ENABLE_WIDGET",
"latitude": 37.7749, # San Francisco latitude
"longitude": -122.4194, # San Francisco longitude
"languageCode": "en_US" # Optional: language for results
}
}] # 👈 ADD GOOGLE MAPS WITH LOCATION
resp = litellm.completion(
model="vertex_ai/gemini-2.0-flash",
messages=[{"role": "user", "content": "What restaurants are nearby?"}],
tools=tools,
)
print(resp)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
<Tabs>
<TabItem value="openai" label="OpenAI Python SDK">
**Basic Usage - Enable Widget Only**
```python showLineNumbers
from openai import OpenAI
client = OpenAI(
api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
base_url="http://0.0.0.0:4000/v1/" # point to litellm proxy
)
response = client.chat.completions.create(
model="gemini-2.0-flash",
messages=[{"role": "user", "content": "What restaurants are nearby?"}],
tools=[{"googleMaps": {"enableWidget": "ENABLE_WIDGET"}}],
)
print(response)
```
**With Location Data**
```python showLineNumbers
from openai import OpenAI
client = OpenAI(
api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
base_url="http://0.0.0.0:4000/v1/" # point to litellm proxy
)
response = client.chat.completions.create(
model="gemini-2.0-flash",
messages=[{"role": "user", "content": "What restaurants are nearby?"}],
tools=[{
"googleMaps": {
"enableWidget": "ENABLE_WIDGET",
"latitude": 37.7749, # San Francisco latitude
"longitude": -122.4194, # San Francisco longitude
"languageCode": "en_US" # Optional: language for results
}
}],
)
print(response)
```
</TabItem>
<TabItem value="curl" label="cURL">
**Basic Usage - Enable Widget Only**
```bash showLineNumbers
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-2.0-flash",
"messages": [
{"role": "user", "content": "What restaurants are nearby?"}
],
"tools": [
{
"googleMaps": {"enableWidget": "ENABLE_WIDGET"}
}
]
}'
```
**With Location Data**
```bash showLineNumbers
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-2.0-flash",
"messages": [
{"role": "user", "content": "What restaurants are nearby?"}
],
"tools": [
{
"googleMaps": {
"enableWidget": "ENABLE_WIDGET",
"latitude": 37.7749,
"longitude": -122.4194,
"languageCode": "en_US"
}
}
]
}'
```
</TabItem>
</Tabs>
</TabItem>
</Tabs>
#### **Moving from Vertex AI SDK to LiteLLM (GROUNDING)**
@ -824,11 +981,158 @@ curl http://0.0.0.0:4000/v1/chat/completions \
### **Context Caching**
Use Vertex AI context caching is supported by calling provider api directly. (Unified Endpoint support coming soon.).
#### Unified Endpoint
Use Vertex AI context caching in the same way as [**Google AI Studio - Context Caching**](../providers/gemini.md#context-caching)
##### Example usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
for _ in range(2):
resp = completion(
model="vertex_ai/gemini-2.5-pro",
messages=[
# System Message
{
"role": "system",
"content": [
{
"type": "text",
"text": "Here is the full text of a complex legal agreement" * 4000,
"cache_control": {"type": "ephemeral"}, # 👈 KEY CHANGE
}
],
},
# marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
{
"role": "user",
"content": [
{
"type": "text",
"text": "What are the key terms and conditions in this agreement?",
"cache_control": {"type": "ephemeral"},
}
],
}]
)
print(resp.usage) # 👈 2nd usage block will be less, since cached tokens used
```
</TabItem>
<TabItem value="sdk-ttl" label="SDK with Custom TTL">
```python
from litellm import completion
# Cache for 2 hours (7200 seconds)
resp = completion(
model="vertex_ai/gemini-2.5-pro",
messages=[
{
"role": "system",
"content": [
{
"type": "text",
"text": "Here is the full text of a complex legal agreement" * 4000,
"cache_control": {
"type": "ephemeral",
"ttl": "7200s" # 👈 Cache for 2 hours
},
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "What are the key terms and conditions in this agreement?",
"cache_control": {
"type": "ephemeral",
"ttl": "3600s" # 👈 This TTL will be ignored (first one is used)
},
}
],
}
]
)
print(resp.usage)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: gemini-2.5-pro
litellm_params:
model: vertex_ai/gemini-2.5-pro
vertex_project: "project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json"
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gemini-2.5-flash",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "Long cache message (must be >= 1024 tokens)",
"cache_control": {
"type": "ephemeral",
"ttl": "7200s"
}
}
]
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "What is the text about?"
}
]
}
]
}'
```
</TabItem>
</Tabs>
#### Calling provider api directly
[**Go straight to provider**](../pass_through/vertex_ai.md#context-caching)
#### 1. Create the Cache
##### 1. Create the Cache
First, create the cache by sending a `POST` request to the `cachedContents` endpoint via the LiteLLM proxy.
@ -854,7 +1158,7 @@ curl http://0.0.0.0:4000/vertex_ai/v1/projects/{project_id}/locations/{location}
</TabItem>
</Tabs>
#### 2. Get the Cache Name from the Response
##### 2. Get the Cache Name from the Response
Vertex AI will return a response containing the `name` of the cached content. This name is the identifier for your cached data.
@ -873,7 +1177,7 @@ Vertex AI will return a response containing the `name` of the cached content. Th
}
```
#### 3. Use the Cached Content
##### 3. Use the Cached Content
Use the `name` from the response as `cachedContent` or `cached_content` in subsequent API calls to reuse the cached information. This is passed in the body of your request to `/chat/completions`.

2
docs/my-website/docs/providers/vertex_batch.md
View file

@ -1,7 +1,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## **Batch APIs**
# Vertex Batch APIs
Just add the following Vertex env vars to your environment.

110
docs/my-website/docs/providers/vertex_partner.md
View file

@ -16,7 +16,6 @@ import TabItem from '@theme/TabItem';
| AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) |
| Qwen | `vertex_ai/qwen/*` | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) |
| OpenAI (GPT-OSS) | `vertex_ai/openai/gpt-oss-*` | [Vertex AI - GPT-OSS Models](https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/) |
| Model Garden | `vertex_ai/openai/{MODEL_ID}` or `vertex_ai/{MODEL_ID}` | [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
## Vertex AI - Anthropic (Claude)
@ -793,112 +792,3 @@ curl http://0.0.0.0:4000/v1/chat/completions \
</TabItem>
</Tabs>
## Model Garden
:::tip
All OpenAI compatible models from Vertex Model Garden are supported.
:::
#### Using Model Garden
**Almost all Vertex Model Garden models are OpenAI compatible.**
<Tabs>
<TabItem value="openai" label="OpenAI Compatible Models">
| Property | Details |
|----------|---------|
| Provider Route | `vertex_ai/openai/{MODEL_ID}` |
| Vertex Documentation | [Model Garden LiteLLM Inference](https://github.com/GoogleCloudPlatform/generative-ai/blob/main/open-models/use-cases/model_garden_litellm_inference.ipynb), [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
| Supported Operations | `/chat/completions`, `/embeddings` |
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
## set ENV variables
os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
response = completion(
model="vertex_ai/openai/<your-endpoint-id>",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: llama3-1-8b-instruct
litellm_params:
model: vertex_ai/openai/5464397967697903616
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-east-1"
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "llama3-1-8b-instruct", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="non-openai" label="Non-OpenAI Compatible Models">
```python
from litellm import completion
import os
## set ENV variables
os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
response = completion(
model="vertex_ai/<your-endpoint-id>",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
</Tabs>

229
docs/my-website/docs/providers/vertex_self_deployed.md Normal file
View file

@ -0,0 +1,229 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Vertex AI - Self Deployed Models
Deploy and use your own models on Vertex AI through Model Garden or custom endpoints.
## Model Garden
:::tip
All OpenAI compatible models from Vertex Model Garden are supported.
:::
### Using Model Garden
**Almost all Vertex Model Garden models are OpenAI compatible.**
<Tabs>
<TabItem value="openai" label="OpenAI Compatible Models">
| Property | Details |
|----------|---------|
| Provider Route | `vertex_ai/openai/{MODEL_ID}` |
| Vertex Documentation | [Model Garden LiteLLM Inference](https://github.com/GoogleCloudPlatform/generative-ai/blob/main/open-models/use-cases/model_garden_litellm_inference.ipynb), [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
| Supported Operations | `/chat/completions`, `/embeddings` |
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
## set ENV variables
os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
response = completion(
model="vertex_ai/openai/<your-endpoint-id>",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: llama3-1-8b-instruct
litellm_params:
model: vertex_ai/openai/5464397967697903616
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-east-1"
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "llama3-1-8b-instruct", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="non-openai" label="Non-OpenAI Compatible Models">
```python
from litellm import completion
import os
## set ENV variables
os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
response = completion(
model="vertex_ai/<your-endpoint-id>",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
</Tabs>
## Gemma Models (Custom Endpoints)
Deploy Gemma models on custom Vertex AI prediction endpoints with OpenAI-compatible format.
| Property | Details |
|----------|---------|
| Provider Route | `vertex_ai/gemma/{MODEL_NAME}` |
| Vertex Documentation | [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions) |
| Required Parameter | `api_base` - Full prediction endpoint URL |
**Proxy Usage:**
**1. Add to config.yaml**
```yaml
model_list:
- model_name: gemma-model
litellm_params:
model: vertex_ai/gemma/gemma-3-12b-it-1222199011122
api_base: https://ENDPOINT.us-central1-PROJECT.prediction.vertexai.goog/v1/projects/PROJECT_ID/locations/us-central1/endpoints/ENDPOINT_ID:predict
vertex_project: "my-project-id"
vertex_location: "us-central1"
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
```
**3. Test it**
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemma-model",
"messages": [{"role": "user", "content": "What is machine learning?"}],
"max_tokens": 100
}'
```
**SDK Usage:**
```python
from litellm import completion
response = completion(
model="vertex_ai/gemma/gemma-3-12b-it-1222199011122",
messages=[{"role": "user", "content": "What is machine learning?"}],
api_base="https://ENDPOINT.us-central1-PROJECT.prediction.vertexai.goog/v1/projects/PROJECT_ID/locations/us-central1/endpoints/ENDPOINT_ID:predict",
vertex_project="my-project-id",
vertex_location="us-central1",
)
```
## MedGemma Models (Custom Endpoints)
Deploy MedGemma models on custom Vertex AI prediction endpoints with OpenAI-compatible format. MedGemma models use the same `vertex_ai/gemma/` route.
| Property | Details |
|----------|---------|
| Provider Route | `vertex_ai/gemma/{MODEL_NAME}` |
| Vertex Documentation | [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions) |
| Required Parameter | `api_base` - Full prediction endpoint URL |
**Proxy Usage:**
**1. Add to config.yaml**
```yaml
model_list:
- model_name: medgemma-model
litellm_params:
model: vertex_ai/gemma/medgemma-2b-v1
api_base: https://ENDPOINT.us-central1-PROJECT.prediction.vertexai.goog/v1/projects/PROJECT_ID/locations/us-central1/endpoints/ENDPOINT_ID:predict
vertex_project: "my-project-id"
vertex_location: "us-central1"
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
```
**3. Test it**
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "medgemma-model",
"messages": [{"role": "user", "content": "What are the symptoms of hypertension?"}],
"max_tokens": 100
}'
```
**SDK Usage:**
```python
from litellm import completion
response = completion(
model="vertex_ai/gemma/medgemma-2b-v1",
messages=[{"role": "user", "content": "What are the symptoms of hypertension?"}],
api_base="https://ENDPOINT.us-central1-PROJECT.prediction.vertexai.goog/v1/projects/PROJECT_ID/locations/us-central1/endpoints/ENDPOINT_ID:predict",
vertex_project="my-project-id",
vertex_location="us-central1",
)
```

196
docs/my-website/docs/providers/wandb_inference.md Normal file
View file

@ -0,0 +1,196 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Weights & Biases Inference
https://weave-docs.wandb.ai/quickstart-inference
:::tip
Litellm provides support to all models from W&B Inference service. To use a model, set `model=wandb/<any-model-on-wandb-inference-dashboard>` as a prefix for litellm requests. The full list of supported models is provided at https://docs.wandb.ai/guides/inference/models/
:::
## API Key
You can get an API key for W&B Inference at - https://wandb.ai/authorize
```python
import os
# env variable
os.environ['WANDB_API_KEY']
```
## Sample Usage: Text Generation
```python
from litellm import completion
import os
os.environ['WANDB_API_KEY'] = "insert-your-wandb-api-key"
response = completion(
model="wandb/Qwen/Qwen3-235B-A22B-Instruct-2507",
messages=[
{
"role": "user",
"content": "What character was Wall-e in love with?",
}
],
max_tokens=10,
response_format={ "type": "json_object" },
seed=123,
temperature=0.6, # either set temperature or `top_p`
top_p=0.01, # to get as deterministic results as possible
)
print(response)
```
## Sample Usage - Streaming
```python
from litellm import completion
import os
os.environ['WANDB_API_KEY'] = ""
response = completion(
model="wandb/Qwen/Qwen3-235B-A22B-Instruct-2507",
messages=[
{
"role": "user",
"content": "What character was Wall-e in love with?",
}
],
stream=True,
max_tokens=10,
response_format={ "type": "json_object" },
seed=123,
temperature=0.6, # either set temperature or `top_p`
top_p=0.01, # to get as deterministic results as possible
)
for chunk in response:
print(chunk)
```
:::tip
The above examples may not work if the model has been taken offline. Check the full list of available models at https://docs.wandb.ai/guides/inference/models/.
:::
## Usage with LiteLLM Proxy Server
Here's how to call a W&B Inference model with the LiteLLM Proxy Server
1. Modify the config.yaml
```yaml
model_list:
- model_name: my-model
litellm_params:
model: wandb/<your-model-name> # add wandb/ prefix to use W&B Inference as provider
api_key: api-key # api key to send your model
```
2. Start the proxy
```bash
$ litellm --config /path/to/config.yaml
```
3. Send Request to LiteLLM Proxy Server
<Tabs>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="litellm-proxy-key", # pass litellm proxy key, if you're using virtual keys
base_url="http://0.0.0.0:4000" # litellm-proxy-base url
)
response = client.chat.completions.create(
model="my-model",
messages = [
{
"role": "user",
"content": "What character was Wall-e in love with?"
}
],
)
print(response)
```
</TabItem>
<TabItem value="curl" label="curl">
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: litellm-proxy-key' \
--header 'Content-Type: application/json' \
--data '{
"model": "my-model",
"messages": [
{
"role": "user",
"content": "What character was Wall-e in love with?"
}
],
}'
```
</TabItem>
</Tabs>
## Supported Parameters
The W&B Inference provider supports the following parameters:
### Chat Completion Parameters
| Parameter | Type | Description |
| --------- | ---- | ----------- |
| frequency_penalty | number | Penalizes new tokens based on their frequency in the text |
| function_call | string/object | Controls how the model calls functions |
| functions | array | List of functions for which the model may generate JSON inputs |
| logit_bias | map | Modifies the likelihood of specified tokens |
| max_tokens | integer | Maximum number of tokens to generate |
| n | integer | Number of completions to generate |
| presence_penalty | number | Penalizes tokens based on if they appear in the text so far |
| response_format | object | Format of the response, e.g., `{"type": "json"}` |
| seed | integer | Sampling seed for deterministic results |
| stop | string/array | Sequences where the API will stop generating tokens |
| stream | boolean | Whether to stream the response |
| temperature | number | Controls randomness (0-2) |
| top_p | number | Controls nucleus sampling |
## Error Handling
The integration uses the standard LiteLLM error handling. Further, here's a list of commonly encountered errors with the W&B Inference API -
| Error Code | Message | Cause | Solution |
| ---------- | ------- | ----- | -------- |
| 401 | Authentication failed | Your authentication credentials are incorrect or your W&B project entity and/or name are incorrect. | Ensure you're using the correct API key and that your W&B project name and entity are correct. |
| 403 | Country, region, or territory not supported | Accessing the API from an unsupported location. | Please see [Geographic restrictions](https://docs.wandb.ai/guides/inference/usage-limits/#geographic-restrictions) |
| 429 | Concurrency limit reached for requests | Too many concurrent requests. | Reduce the number of concurrent requests or increase your limits. For more information, see [Usage information and limits](https://docs.wandb.ai/guides/inference/usage-limits/). |
| 429 | You exceeded your current quota, please check your plan and billing details | Out of credits or reached monthly spending cap. | Get more credits or increase your limits. For more information, see [Usage information and limits](https://docs.wandb.ai/guides/inference/usage-limits/). |
| 429 | W&B Inference isn't available for personal accounts. | Switch to a non-personal account. | Follow [the instructions below](#error-429-personal-entities-unsupported) for a work around. |
| 500 | The server had an error while processing your request | Internal server error. | Retry after a brief wait and contact support if it persists. |
| 503 | The engine is currently overloaded, please try again later | Server is experiencing high traffic. | Retry your request after a short delay. |
### Error 429: Personal entities unsupported
The user is on a personal account, which doesn't have access to W&B Inference. If one isn't available, create a Team to create a non-personal account.
Once done, add the `openai-project` header to your request as shown below:
```python
response = completion(
model="...",
extra_headers={"openai-project": "team_name/project_name"},
...
```
For more information, see [Personal entities unsupported](https://docs.wandb.ai/guides/inference/usage-limits/#personal-entities-unsupported).
You can find more ways of using custom headers with LiteLLM here - https://docs.litellm.ai/docs/proxy/request_headers.

17
docs/my-website/docs/proxy/admin_ui_sso.md
View file

@ -81,6 +81,23 @@ MICROSOFT_TENANT="5a39737
http://localhost:4000/sso/callback
```
**Using App Roles for User Permissions**
You can assign user roles directly from Entra ID using App Roles. LiteLLM will automatically read the app roles from the JWT token and assign the corresponding role to the user.
Supported roles:
- `proxy_admin` - Admin over the platform
- `proxy_admin_viewer` - Can login, view all keys, view all spend (read-only)
- `internal_user` - Normal user. Can login, view spend and depending on team-member permissions - view/create/delete their own keys.
To set up app roles:
1. Navigate to your App Registration on https://portal.azure.com/
2. Go to "App roles" and create a new app role
3. Use one of the supported role names above (e.g., `proxy_admin`)
4. Assign users to these roles in your Enterprise Application
5. When users sign in via SSO, LiteLLM will automatically assign them the corresponding role
</TabItem>
<TabItem value="Generic" label="Generic SSO Provider">

2
docs/my-website/docs/proxy/caching.md
View file

@ -278,6 +278,8 @@ Set either `REDIS_URL` or the `REDIS_HOST` in your os environment, to enable cac
REDIS_HOST = "" # REDIS_HOST='redis-18841.c274.us-east-1-3.ec2.cloud.redislabs.com'
REDIS_PORT = "" # REDIS_PORT='18841'
REDIS_PASSWORD = "" # REDIS_PASSWORD='liteLlmIsAmazing'
REDIS_USERNAME = "" # REDIS_USERNAME='my-redis-username' [OPTIONAL] if your redis server requires a username
REDIS_SSL = "True" # REDIS_SSL='True' to enable SSL by default is False
```
**Additional kwargs**

15
docs/my-website/docs/proxy/config_settings.md
View file

@ -224,6 +224,7 @@ router_settings:
| service_account_settings | List[Dict[str, Any]] | Set `service_account_settings` if you want to create settings that only apply to service account keys (Doc on service accounts)[./service_accounts.md] |
| image_generation_model | str | The default model to use for image generation - ignores model set in request |
| store_model_in_db | boolean | If true, enables storing model + credential information in the DB. |
| supported_db_objects | List[str] | Fine-grained control over which object types to load from the database when `store_model_in_db` is True. Available types: `"models"`, `"mcp"`, `"guardrails"`, `"vector_stores"`, `"pass_through_endpoints"`, `"prompts"`, `"model_cost_map"`. If not set, all object types are loaded (default behavior). Example: `supported_db_objects: ["mcp"]` to only load MCP servers from DB. |
| store_prompts_in_spend_logs | boolean | If true, allows prompts and responses to be stored in the spend logs table. |
| max_request_size_mb | int | The maximum size for requests in MB. Requests above this size will be rejected. |
| max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. |
@ -352,7 +353,10 @@ router_settings:
| AGENTOPS_SERVICE_NAME | Service Name for AgentOps logging integration
| AISPEND_ACCOUNT_ID | Account ID for AI Spend
| AISPEND_API_KEY | API Key for AI Spend
| AIOHTTP_CONNECTOR_LIMIT | Connection limit for aiohttp connector. When set to 0, no limit is applied. **Default is 0**
| AIOHTTP_KEEPALIVE_TIMEOUT | Keep-alive timeout for aiohttp connections in seconds. **Default is 120**
| AIOHTTP_TRUST_ENV | Flag to enable aiohttp trust environment. When this is set to True, aiohttp will respect HTTP(S)_PROXY env vars. **Default is False**
| AIOHTTP_TTL_DNS_CACHE | DNS cache time-to-live for aiohttp in seconds. **Default is 300**
| ALLOWED_EMAIL_DOMAINS | List of email domains allowed for access
| ARIZE_API_KEY | API key for Arize platform integration
| ARIZE_SPACE_KEY | Space key for Arize platform
@ -505,6 +509,8 @@ router_settings:
| EMAIL_SIGNATURE | Custom HTML footer/signature for all emails. Can include HTML tags for formatting and links.
| EMAIL_SUBJECT_INVITATION | Custom subject template for invitation emails.
| EMAIL_SUBJECT_KEY_CREATED | Custom subject template for key creation emails.
| ENKRYPTAI_API_BASE | Base URL for EnkryptAI Guardrails API. **Default is https://api.enkryptai.com**
| ENKRYPTAI_API_KEY | API key for EnkryptAI Guardrails service
| EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False**
| FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4
| FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16
@ -628,6 +634,7 @@ router_settings:
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
| LITELLM_SALT_KEY | Salt key for encryption in LiteLLM
| LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections.
| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
| LITELLM_TOKEN | Access token for LiteLLM integration
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
@ -773,6 +780,8 @@ router_settings:
| USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption
| USE_PRISMA_MIGRATE | Flag to use prisma migrate instead of prisma db push. Recommended for production environments.
| WEBHOOK_URL | URL for receiving webhooks from external services
| SPEND_LOG_RUN_LOOPS | Constant for setting how many runs of 1000 batch deletes should spend_log_cleanup task run |
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000 |
| COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY | Maximum size for CoroutineChecker in-memory cache. Default is 1000 |
| SPEND_LOG_RUN_LOOPS | Constant for setting how many runs of 1000 batch deletes should spend_log_cleanup task run
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000
| COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY | Maximum size for CoroutineChecker in-memory cache. Default is 1000
| DEFAULT_SHARED_HEALTH_CHECK_TTL | Time-to-live in seconds for cached health check results in shared health check mode. Default is 300 (5 minutes)
| DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL | Time-to-live in seconds for health check lock in shared health check mode. Default is 60 (1 minute)

4
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -8,6 +8,10 @@ Track spend for keys, users, and teams across 100+ LLMs.
LiteLLM automatically tracks spend for all known models. See our [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
:::tip Keep Pricing Data Updated
[Sync model pricing data from GitHub](../sync_models_github.md) to ensure accurate cost tracking.
:::
### How to Track Spend with LiteLLM
**Step 1**

4
docs/my-website/docs/proxy/custom_prompt_management.md
View file

@ -127,7 +127,9 @@ client = OpenAI(
response = client.chat.completions.create(
model="gemini-1.5-pro",
messages=[{"role": "user", "content": "hi"}],
prompt_id="1234"
extra_body={
"prompt_id": "1234"
}
)
print(response.choices[0].message.content)

43
docs/my-website/docs/proxy/deploy.md
View file

@ -715,6 +715,25 @@ docker run ghcr.io/berriai/litellm:main-stable
```
### Restart Workers After N Requests
Use this to mitigate memory growth by recycling workers after a fixed number of requests. When set, each worker restarts after completing the specified number of requests. Defaults to disabled when unset.
Usage Examples:
```shell showLineNumbers title="docker run (CLI flag)"
docker run ghcr.io/berriai/litellm:main-stable \
--max_requests_before_restart 10000
```
Or set via environment variable:
```shell showLineNumbers title="Environment Variable"
export MAX_REQUESTS_BEFORE_RESTART=10000
docker run ghcr.io/berriai/litellm:main-stable
```
### 5. config.yaml file on s3, GCS Bucket Object/url
Use this if you cannot mount a config file on your deployment service (example - AWS Fargate, Railway etc)
@ -769,6 +788,30 @@ docker run --name litellm-proxy \
## Platform-specific Guide
<Tabs>
<TabItem value="AWS ECS" label="AWS ECS - Elastic Container Service">
### Terraform-based ECS Deployment
LiteLLM maintains a dedicated Terraform tutorial for deploying the proxy on ECS. Follow the step-by-step guide in the [litellm-ecs-deployment repository](https://github.com/BerriAI/litellm-ecs-deployment) to provision the required ECS services, task definitions, and supporting AWS resources.
1. Clone the tutorial repository to review the Terraform modules and variables.
```bash
git clone https://github.com/BerriAI/litellm-ecs-deployment.git
cd litellm-ecs-deployment
```
2. Initialize and validate the Terraform project before applying it to your chosen workspace/account.
```bash
terraform init
terraform plan
terraform apply
```
3. Once `terraform apply` completes, do `./build.sh` to push the repository on ECR and update the ECS cluster. Use that endpoint (port `4000` by default) for API requests to your LiteLLM proxy.
</TabItem>
<TabItem value="AWS EKS" label="AWS EKS - Kubernetes">
### Kubernetes (AWS EKS)

3
docs/my-website/docs/proxy/dynamic_rate_limit.md
View file

@ -141,6 +141,7 @@ litellm_settings:
"dev": 0.1 # 10% reserved for development (1 RPM)
priority_reservation_settings:
default_priority: 0 # Weight (0%) assigned to keys without explicit priority metadata
saturation_threshold: 0.50 # A model is saturated if it has hit 50% of its RPM limit
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
@ -156,6 +157,8 @@ general_settings:
`priority_reservation_settings`: Object (Optional)
- **default_priority (float)**: Weight/percentage (0.0 to 1.0) assigned to API keys that have no priority metadata set (defaults to 0.5)
- **saturation_threshold (float)**: Saturation level (0.0 to 1.0) at which strict priority enforcement begins for a model. Saturation is calculated as `max(current_rpm/max_rpm, current_tpm/max_tpm)`. Below this threshold, generous mode allows priority borrowing from unused capacity. Above this threshold, strict mode enforces normalized priority limits.
- Example: When model usage is low, keys can use more than their allocated share. When model usage is high, keys are strictly limited to their allocated share.
**Start Proxy**

276
docs/my-website/docs/proxy/guardrails/enkryptai.md Normal file
View file

@ -0,0 +1,276 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# EnkryptAI Guardrails
LiteLLM supports EnkryptAI guardrails for content moderation and safety checks on LLM inputs and outputs.
## Quick Start
### 1. Define Guardrails on your LiteLLM config.yaml
Define your guardrails under the `guardrails` section:
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "enkryptai-guard"
litellm_params:
guardrail: enkryptai
mode: "pre_call"
api_key: os.environ/ENKRYPTAI_API_KEY
detectors:
toxicity:
enabled: true
nsfw:
enabled: true
pii:
enabled: true
entities: ["email", "phone", "secrets"]
injection_attack:
enabled: true
```
#### Supported values for `mode`
- `pre_call` - Run **before** LLM call, on **input**
- `post_call` - Run **after** LLM call, on **output**
- `during_call` - Run **during** LLM call, on **input**. Same as `pre_call` but runs in parallel as LLM call
#### Available Detectors
EnkryptAI supports multiple content detection types:
- **toxicity** - Detect toxic language
- **nsfw** - Detect NSFW (Not Safe For Work) content
- **pii** - Detect personally identifiable information
- Configure entities: `["pii", "email", "phone", "secrets", "ip_address", "url"]`
- **injection_attack** - Detect prompt injection attempts
- **keyword_detector** - Detect custom keywords/phrases
- **policy_violation** - Detect policy violations
- **bias** - Detect biased content
- **sponge_attack** - Detect sponge attacks
### 2. Set Environment Variables
```bash
export ENKRYPTAI_API_KEY="your-api-key"
```
### 3. Start LiteLLM Gateway
```shell
litellm --config config.yaml --detailed_debug
```
### 4. Test Request
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
<Tabs>
<TabItem label="Successful Call" value="allowed">
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{"role": "user", "content": "Hello, how can you help me today?"}
],
"guardrails": ["enkryptai-guard"]
}'
```
**Response: HTTP 200 Success**
Content passes all detector checks and is allowed through.
</TabItem>
<TabItem label="Unsuccessful Call" value="not-allowed">
Expect this to fail if content violates detector policies:
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{"role": "user", "content": "My email is test@example.com and my SSN is 123-45-6789"}
],
"guardrails": ["enkryptai-guard"]
}'
```
**Expected Response on Failure: HTTP 400 Error**
```json
{
"error": {
"message": {
"error": "Content blocked by EnkryptAI guardrail",
"detected": true,
"violations": ["pii"],
"response": {
"summary": {
"pii": 1
},
"details": {
"pii": {
"detected": ["email", "ssn"]
}
}
}
},
"type": "None",
"param": "None",
"code": "400"
}
}
```
</TabItem>
</Tabs>
## Video Walkthrough
<iframe width="840" height="500" src="https://www.loom.com/embed/ff222211e0864937aee4aeef0f28c3b7" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## Advanced Configuration
### Using Custom Policies
You can specify a custom EnkryptAI policy:
```yaml
guardrails:
- guardrail_name: "enkryptai-custom"
litellm_params:
guardrail: enkryptai
mode: "pre_call"
api_key: os.environ/ENKRYPTAI_API_KEY
policy_name: "my-custom-policy" # Sent via x-enkrypt-policy header
detectors:
toxicity:
enabled: true
```
### Using Deployments
Specify an EnkryptAI deployment:
```yaml
guardrails:
- guardrail_name: "enkryptai-deployment"
litellm_params:
guardrail: enkryptai
mode: "pre_call"
api_key: os.environ/ENKRYPTAI_API_KEY
deployment_name: "production" # Sent via X-Enkrypt-Deployment header
detectors:
toxicity:
enabled: true
```
### Monitor Mode (Logging Without Blocking)
Set `block_on_violation: false` to log violations without blocking requests:
```yaml
guardrails:
- guardrail_name: "enkryptai-monitor"
litellm_params:
guardrail: enkryptai
mode: "pre_call"
api_key: os.environ/ENKRYPTAI_API_KEY
block_on_violation: false # Log violations but don't block
detectors:
toxicity:
enabled: true
nsfw:
enabled: true
```
In monitor mode, all violations are logged but requests are never blocked.
### Input and Output Guardrails
Configure separate guardrails for input and output:
```yaml
guardrails:
# Input guardrail
- guardrail_name: "enkryptai-input"
litellm_params:
guardrail: enkryptai
mode: "pre_call"
api_key: os.environ/ENKRYPTAI_API_KEY
detectors:
pii:
enabled: true
entities: ["email", "phone", "ssn"]
injection_attack:
enabled: true
# Output guardrail
- guardrail_name: "enkryptai-output"
litellm_params:
guardrail: enkryptai
mode: "post_call"
api_key: os.environ/ENKRYPTAI_API_KEY
detectors:
toxicity:
enabled: true
nsfw:
enabled: true
```
## Configuration Options
| Parameter | Type | Description | Default |
|-----------|------|-------------|---------|
| `api_key` | string | EnkryptAI API key | `ENKRYPTAI_API_KEY` env var |
| `api_base` | string | EnkryptAI API base URL | `https://api.enkryptai.com` |
| `policy_name` | string | Custom policy name (sent via `x-enkrypt-policy` header) | None |
| `deployment_name` | string | Deployment name (sent via `X-Enkrypt-Deployment` header) | None |
| `detectors` | object | Detector configuration | `{}` |
| `block_on_violation` | boolean | Block requests on violations | `true` |
| `mode` | string | When to run: `pre_call`, `post_call`, or `during_call` | Required |
## Observability
EnkryptAI guardrail logs include:
- **guardrail_status**: `success`, `guardrail_intervened`, or `guardrail_failed_to_respond`
- **guardrail_provider**: `enkryptai`
- **guardrail_json_response**: Full API response with detection details
- **duration**: Time taken for guardrail check
- **start_time** and **end_time**: Timestamps
These logs are available through your configured LiteLLM logging callbacks.
## Error Handling
The guardrail handles errors gracefully:
- **API Failures**: Logs error and raises exception
- **Rate Limits (429)**: Logs error and raises exception
- **Invalid Configuration**: Raises `ValueError` on initialization
Set `block_on_violation: false` to continue processing even when violations are detected (monitor mode).
## Support
For more information about EnkryptAI:
- Documentation: [https://docs.enkryptai.com](https://docs.enkryptai.com)
- Website: [https://enkryptai.com](https://enkryptai.com)

21
docs/my-website/docs/proxy/health.md
View file

@ -9,13 +9,32 @@ Use this to health check all LLMs defined in your config.yaml
| `/health/readiness` | **Load balancer health checks** | Ready to accept traffic - includes DB connection status |
| `/health` | **Model health monitoring** | Comprehensive LLM model health - makes actual API calls |
| `/health/services` | **Service debugging** | Check specific integrations (datadog, langfuse, etc.) |
| `/health/shared-status` | **Multi-pod coordination** | Monitor shared health check state across pods |
## Summary
The proxy exposes:
* a /health endpoint which returns the health of the LLM APIs
* a /health/readiness endpoint for returning if the proxy is ready to accept requests
* a /health/liveliness endpoint for returning if the proxy is alive
* a /health/liveliness endpoint for returning if the proxy is alive
* a /health/shared-status endpoint for monitoring shared health check coordination across pods
## Shared Health Check State
When running multiple LiteLLM proxy pods, you can enable shared health check state to coordinate health checks across pods and avoid duplicate API calls. This is especially beneficial for expensive models like Gemini 2.5-pro.
**Key Benefits:**
- Reduces duplicate health checks across pods
- Saves costs on expensive model API calls
- Reduces monitoring noise and logging
- Improves resource efficiency
**Requirements:**
- Redis for shared state coordination
- Background health checks enabled
- Multiple proxy pods
For detailed configuration and usage, see [Shared Health Check State](./shared_health_check.md).
## `/health`
#### Request

4
docs/my-website/docs/proxy/model_management.md
View file

@ -19,6 +19,10 @@ model_list:
Retrieve detailed information about each model listed in the `/model/info` endpoint, including descriptions from the `config.yaml` file, and additional model info (e.g. max tokens, cost per input token, etc.) pulled from the model_info you set and the [litellm model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Sensitive details like API keys are excluded for security purposes.
:::tip Sync Model Data
Keep your model pricing data up to date by [syncing models from GitHub](../sync_models_github.md).
:::
<Tabs
defaultValue="curl"
values={[

10
docs/my-website/docs/proxy/prod.md
View file

@ -71,6 +71,16 @@ Use this Docker `CMD`. This will start the proxy with 1 Uvicorn Async Worker
CMD ["--port", "4000", "--config", "./proxy_server_config.yaml"]
```
> Optional: If you observe gradual memory growth under sustained load, consider recycling workers after a fixed number of requests to mitigate leaks. Set this via CLI or environment variable:
```shell
# CLI
CMD ["--port", "4000", "--config", "./proxy_server_config.yaml", "--max_requests_before_restart", "10000"]
# or ENV (for deployment manifests / containers)
export MAX_REQUESTS_BEFORE_RESTART=10000
```
## 4. Use Redis 'port','host', 'password'. NOT 'redis_url'

30
docs/my-website/docs/proxy/prometheus.md
View file

@ -221,6 +221,8 @@ litellm_settings:
2. Make a request with the custom metadata labels
<Tabs>
<TabItem value="Curl" label="Curl Request">
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
@ -244,6 +246,34 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
}
}'
```
</TabItem>
<TabItem value="key" label="on Key">
```bash
curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {
"foo": "hello world"
}
}'
```
</TabItem>
<TabItem value="team" label="on Team">
```bash
curl -L -X POST 'http://0.0.0.0:4000/team/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {
"foo": "hello world"
}
}'
```
</TabItem>
</Tabs>
3. Check your `/metrics` endpoint for the custom metrics

354
docs/my-website/docs/proxy/security_encryption_faq.md Normal file
View file

@ -0,0 +1,354 @@
# LiteLLM Self-Hosted Security & Encryption FAQ
## Data in Transit Encryption
### Does the product encrypt data in transit?
**Yes**, LiteLLM encrypts data in transit using TLS/SSL.
### Available in both OSS and Enterprise?
**Yes**, TLS encryption is available in both Open Source and Enterprise versions.
### In transit between the calling client and the product?
**Yes**, HTTPS/TLS is supported through SSL certificate configuration.
**Configuration:**
```bash
# CLI
litellm --ssl_keyfile_path /path/to/key.pem --ssl_certfile_path /path/to/cert.pem
# Environment Variables
export SSL_KEYFILE_PATH="/path/to/key.pem"
export SSL_CERTFILE_PATH="/path/to/cert.pem"
```
**Documentation Reference:** `docs/my-website/docs/guides/security_settings.md`
### In transit between the product and the LLM providers?
**Yes**, all connections to LLM providers use TLS encryption by default.
**Implementation Details:**
- Uses Python's `ssl.create_default_context()`
- Leverages HTTPX and aiohttp libraries with SSL/TLS enabled
- Uses certifi CA bundle by default for SSL verification
**Code Reference:** `litellm/llms/custom_httpx/http_handler.py` (lines 43-105)
### Are TCP sessions to the LLM providers shared?
**Yes**, TCP connections are pooled and reused.
**Details:**
- Connection pooling is enabled by default
- Default: 1000 max concurrent connections with keepalive
- Sessions are maintained across requests to the same provider
- Reduces overhead of TLS handshakes
**Code Reference:** `litellm/llms/custom_httpx/http_handler.py` (lines 704-712)
### Or does the product negotiate a new TLS session with the same LLM provider for every sequential call?
**No**, TLS sessions are reused through connection pooling. New TLS handshakes are not performed for every request.
### How is it encrypted?
**TLS 1.2 and TLS 1.3**
Uses Python's default SSL context which supports both TLS 1.2 and TLS 1.3. The specific version negotiated depends on:
- Python version
- System SSL library (typically OpenSSL)
- Server capabilities
**Implementation:** `ssl.create_default_context()` in Python
### How are these added to the product's configuration?
#### x.509 Certificate
**Method 1: CLI Arguments**
```bash
litellm --ssl_certfile_path /path/to/certificate.pem
```
**Method 2: Environment Variable**
```bash
export SSL_CERTFILE_PATH="/path/to/certificate.pem"
```
#### Private Key
**Method 1: CLI Arguments**
```bash
litellm --ssl_keyfile_path /path/to/private_key.pem
```
**Method 2: Environment Variable**
```bash
export SSL_KEYFILE_PATH="/path/to/private_key.pem"
```
#### Certificate Bundle/Chain
**For client-to-proxy connections:**
Use standard SSL certificate setup with intermediate certificates bundled in the certfile.
**For proxy-to-LLM provider connections:**
**Method 1: Config YAML**
```yaml
litellm_settings:
ssl_verify: "/path/to/ca_bundle.pem"
```
**Method 2: Environment Variable**
```bash
export SSL_CERT_FILE="/path/to/ca_bundle.pem"
```
**Method 3: Client Certificate Authentication**
```yaml
litellm_settings:
ssl_certificate: "/path/to/client_certificate.pem"
```
or
```bash
export SSL_CERTIFICATE="/path/to/client_certificate.pem"
```
### Documentation Coverage
**Primary Documentation:**
- `docs/my-website/docs/guides/security_settings.md` - SSL/TLS configuration guide
**Additional References:**
- `litellm/proxy/proxy_cli.py` (lines 455-467) - CLI options
- `docs/my-website/docs/completion/http_handler_config.md` - Custom HTTP handler configuration
---
## Data at Rest Encryption
### Does the product encrypt data at rest?
**Partially**. Only specific sensitive data is encrypted at rest.
### What data is stored in encrypted form?
#### Encrypted Data:
1. **LLM API Keys** - Model credentials in `LiteLLM_ProxyModelTable.litellm_params`
2. **Provider Credentials** - Stored in `LiteLLM_CredentialsTable.credential_values`
3. **Configuration Secrets** - Sensitive config values in `LiteLLM_Config` table
4. **Virtual Keys** - When using secret managers (optional feature)
#### NOT Encrypted:
1. **Spend Logs** - Request/response data in `LiteLLM_SpendLogs`
2. **Audit Logs** - Change history in `LiteLLM_AuditLog`
3. **User/Team/Organization Data** - Metadata and configuration
4. **Cached Prompts and Completions** - Cache data is stored in plaintext
### Cached prompts and completions?
**No**, cached prompts and completions are **NOT encrypted**.
Cache backends (Redis, S3, local disk) store data as plaintext JSON.
**Code References:**
- `litellm/caching/redis_cache.py`
- `litellm/caching/s3_cache.py`
- `litellm/caching/caching.py`
### Configuration data?
**Partially encrypted**.
#### What IS Encrypted:
- LLM API keys and credentials in model configurations
- Sensitive values in `LiteLLM_Config` table
- Credential values in `LiteLLM_CredentialsTable`
#### What is NOT Encrypted:
- Model names and aliases
- Rate limits and budget settings
- User/team/organization metadata
- Non-sensitive configuration parameters
**Code Reference:** `litellm/proxy/management_endpoints/model_management_endpoints.py` (lines 275-308)
### Log data?
**No**, log data is **NOT encrypted**.
Log data stored in database tables is in plaintext:
- `LiteLLM_SpendLogs` - Contains request/response data, tokens, spend
- `LiteLLM_ErrorLogs` - Error information
- `LiteLLM_AuditLog` - Audit trail of changes
**Note:** You can disable logging to avoid storing sensitive data:
```yaml
general_settings:
disable_spend_logs: True # Disable writing spend logs to DB
disable_error_logs: True # Disable writing error logs to DB
```
**Documentation:** `docs/my-website/docs/proxy/db_info.md` (lines 52-60)
### Where is it stored?
#### In the DB?
**Yes**, encrypted data is stored in PostgreSQL database.
**Key Tables with Encrypted Data:**
- `LiteLLM_ProxyModelTable` - Model configurations with encrypted API keys
- `LiteLLM_CredentialsTable` - Credential values
- `LiteLLM_Config` - Configuration secrets
**Schema Reference:** `schema.prisma`
#### In the filesystem?
**No**, encrypted data is not stored in the filesystem by default.
**Note:** If using disk cache (`disk_cache_dir`), cached data is stored unencrypted.
#### Somewhere else?
**Optional:** When using secret managers (AWS Secrets Manager, Azure Key Vault, HashiCorp Vault), encrypted data can be stored externally.
**Configuration:**
```yaml
general_settings:
key_management_system: "aws_secret_manager" # or "azure_key_vault", "hashicorp_vault"
```
**Documentation:** `docs/my-website/docs/secret.md`
### How is it encrypted?
**Algorithm:** NaCl SecretBox (XSalsa20-Poly1305 AEAD)
**NOT AES-256** - LiteLLM uses NaCl (Networking and Cryptography Library) which provides:
- XSalsa20 stream cipher
- Poly1305 MAC for authentication
- Equivalent security to AES-256
**Key Derivation:**
1. Takes `LITELLM_SALT_KEY` (or `LITELLM_MASTER_KEY` if salt key not set)
2. Hashes with SHA-256 to derive 256-bit encryption key
3. Uses NaCl SecretBox for authenticated encryption
**Code Reference:** `litellm/proxy/common_utils/encrypt_decrypt_utils.py` (lines 69-112)
**Implementation:**
```python
import hashlib
import nacl.secret
# Derive 256-bit key from salt
hash_object = hashlib.sha256(signing_key.encode())
hash_bytes = hash_object.digest()
# Create SecretBox and encrypt
box = nacl.secret.SecretBox(hash_bytes)
encrypted = box.encrypt(value_bytes)
```
### Setting the Encryption Key
**Required Environment Variable:**
```bash
export LITELLM_SALT_KEY="your-strong-random-key-here"
```
**Important Notes:**
- ⚠️ **Must be set before adding any models**
- ⚠️ **Never change this key** - encrypted data becomes unrecoverable
- ⚠️ Use a strong random key (recommended: https://1password.com/password-generator/)
- If not set, falls back to `LITELLM_MASTER_KEY`
**Documentation:** `docs/my-website/docs/proxy/prod.md` (section 8, lines 184-196)
### Documentation Coverage
**Primary Documentation:**
- `docs/my-website/docs/proxy/prod.md` (section 8) - LITELLM_SALT_KEY setup
- `docs/my-website/docs/secret.md` - Secret management systems
- `docs/my-website/docs/proxy/db_info.md` - Database information
**Additional References:**
- `security.md` - General security measures
- `docs/my-website/docs/data_security.md` - Data privacy overview
- `schema.prisma` - Database schema with encrypted fields
---
## Summary of Security Features
### ✅ Provided Out of the Box
1. **TLS/SSL encryption** for client-to-proxy connections
2. **TLS encryption** for proxy-to-LLM provider connections (with connection pooling)
3. **Encrypted storage** of LLM API keys and credentials
4. **Support for TLS 1.2 and TLS 1.3**
5. **Connection pooling** to reduce TLS handshake overhead
### ⚠️ Important Limitations
1. **Cached data is NOT encrypted** (Redis, S3, disk cache)
2. **Log data is NOT encrypted** (spend logs, audit logs)
3. **Request/response payloads in logs are NOT encrypted**
4. **Uses NaCl SecretBox, NOT AES-256** (equivalent security)
5. **TLS version not explicitly configured** - uses Python/system defaults
### 🔧 Configuration Requirements
**For Production Deployments:**
1. **Set LITELLM_SALT_KEY** before adding any models
2. **Configure SSL certificates** for HTTPS client connections
3. **Consider disabling logs** if they contain sensitive data
4. **Use secret managers** for enhanced security (optional)
5. **Configure CA bundles** if using custom certificates
---
## Quick Start Security Checklist
```bash
# 1. Generate a strong salt key
export LITELLM_SALT_KEY="$(openssl rand -base64 32)"
# 2. Set up SSL certificates (for HTTPS)
export SSL_KEYFILE_PATH="/path/to/private_key.pem"
export SSL_CERTFILE_PATH="/path/to/certificate.pem"
# 3. Configure database
export DATABASE_URL="postgresql://user:password@host:port/dbname"
# 4. (Optional) Disable logs if they contain sensitive data
# Add to config.yaml:
# general_settings:
# disable_spend_logs: True
# disable_error_logs: True
# 5. Start LiteLLM Proxy
litellm --config config.yaml
```
---
## Additional Resources
- **LiteLLM Documentation:** https://docs.litellm.ai/
- **Security Settings Guide:** https://docs.litellm.ai/docs/guides/security_settings
- **Production Deployment:** https://docs.litellm.ai/docs/proxy/prod
- **Secret Management:** https://docs.litellm.ai/docs/secret
For security inquiries: support@berri.ai

310
docs/my-website/docs/proxy/shared_health_check.md Normal file
View file

@ -0,0 +1,310 @@
# Shared Health Check State Across Pods
This feature enables coordination of health checks across multiple LiteLLM proxy pods to avoid duplicate health checks and reduce costs.
## Overview
When running multiple LiteLLM proxy pods (e.g., in Kubernetes), each pod typically runs its own independent health checks on every model. This can result in:
- **Duplicate health checks** across pods
- **Increased costs** for expensive models (e.g., Gemini 2.5-pro)
- **Redundant monitoring/logging noise**
- **Inefficient resource usage**
The shared health check state feature solves this by:
- **Coordinating health checks** across pods using Redis
- **Caching results** with configurable TTL
- **Using distributed locks** to ensure only one pod runs health checks at a time
- **Allowing other pods** to read cached results instead of running redundant checks
## How It Works
### 1. Lock Acquisition
When a pod needs to run health checks:
- It attempts to acquire a Redis lock
- If successful, it runs the health checks
- If failed, it waits briefly and checks for cached results
### 2. Result Caching
After running health checks:
- Results are cached in Redis with a configurable TTL
- Other pods can read these cached results
- Cache includes timestamp and pod ID for tracking
### 3. Fallback Behavior
If Redis is unavailable or cache is expired:
- Pods fall back to running health checks locally
- System continues to function normally
## Configuration
### Enable Shared Health Check
Add to your `proxy_config.yaml`:
```yaml
general_settings:
# Enable background health checks (required)
background_health_checks: true
# Enable shared health check state across pods
use_shared_health_check: true
# Health check interval (seconds)
health_check_interval: 300 # 5 minutes
# Redis configuration (required for shared health check)
litellm_settings:
cache: true
cache_params:
type: redis
host: your-redis-host
port: 6379
password: your-redis-password
```
### Environment Variables
You can also configure using environment variables:
```bash
# Enable shared health check
export USE_SHARED_HEALTH_CHECK=true
# Health check TTL (seconds)
export DEFAULT_SHARED_HEALTH_CHECK_TTL=300
# Lock TTL (seconds)
export DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL=60
```
## Requirements
- **Redis**: Required for shared state coordination
- **Background Health Checks**: Must be enabled (`background_health_checks: true`)
- **Multiple Pods**: Most beneficial with 2+ proxy instances
## API Endpoints
### Check Shared Health Check Status
```bash
GET /health/shared-status
```
Returns information about the shared health check coordination:
```json
{
"shared_health_check_enabled": true,
"status": {
"pod_id": "pod_1703123456789",
"redis_available": true,
"lock_ttl": 60,
"cache_ttl": 300,
"lock_owner": "pod_1703123456788",
"lock_in_progress": true,
"cache_available": true,
"cache_age_seconds": 45.2,
"last_checked_by": "pod_1703123456788"
}
}
```
## Monitoring
### Health Check Status
Monitor the shared health check status to ensure proper coordination:
```bash
curl -H "Authorization: Bearer your-api-key" \
http://your-proxy-host/health/shared-status
```
### Logs
Look for these log messages:
```
INFO: Initialized shared health check manager
INFO: Pod pod_123 acquired health check lock
INFO: Pod pod_123 released health check lock
INFO: Cached health check results for 5 healthy and 0 unhealthy endpoints
DEBUG: Using cached health check results
```
## Troubleshooting
### Common Issues
#### 1. Shared Health Check Not Working
**Symptoms**: Each pod still runs independent health checks
**Solutions**:
- Verify Redis is configured and accessible
- Check that `use_shared_health_check: true` is set
- Ensure `background_health_checks: true` is enabled
- Check Redis connectivity in logs
#### 2. Redis Connection Issues
**Symptoms**: Health checks fall back to local execution
**Solutions**:
- Verify Redis host, port, and credentials
- Check network connectivity between pods and Redis
- Monitor Redis server logs for errors
#### 3. Lock Not Released
**Symptoms**: One pod holds the lock indefinitely
**Solutions**:
- Lock has automatic TTL (default 60 seconds)
- Check pod logs for lock release messages
- Verify Redis TTL settings
### Debug Mode
Enable debug logging to see detailed coordination:
```yaml
general_settings:
set_verbose: true
```
## Performance Impact
### Benefits
- **Reduced API calls**: Only one pod runs health checks per interval
- **Lower costs**: Especially significant for expensive models
- **Better resource utilization**: Less redundant work across pods
- **Cleaner monitoring**: Reduced noise in logs and metrics
### Overhead
- **Redis operations**: Minimal overhead for lock/cache operations
- **Network latency**: Small delay for Redis communication
- **Memory usage**: Negligible additional memory usage
## Best Practices
### 1. Redis Configuration
- Use Redis with persistence enabled
- Configure appropriate memory limits
- Set up Redis monitoring and alerts
### 2. TTL Settings
- Set `health_check_interval` to your desired check frequency
- Use default TTL values unless you have specific requirements
- Consider model-specific timeouts for expensive models
### 3. Monitoring
- Monitor shared health check status endpoint
- Set up alerts for Redis connectivity issues
- Track health check costs and frequency
### 4. Scaling
- Feature works with any number of pods
- More pods = better coordination benefits
- Consider Redis cluster for high availability
## Example Configuration
### Complete Example
```yaml
# proxy_config.yaml
model_list:
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
model_info:
health_check_timeout: 30 # 30 second timeout for health checks
general_settings:
# Enable background health checks
background_health_checks: true
# Enable shared health check coordination
use_shared_health_check: true
# Health check interval (5 minutes)
health_check_interval: 300
# Health check details
health_check_details: true
litellm_settings:
# Redis configuration
cache: true
cache_params:
type: redis
host: redis-cluster.example.com
port: 6379
password: os.environ/REDIS_PASSWORD
ssl: true
```
### Kubernetes Example
```yaml
# deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: litellm-proxy
spec:
replicas: 3 # Multiple pods for coordination
template:
spec:
containers:
- name: litellm-proxy
image: ghcr.io/berriai/litellm:latest
env:
- name: USE_SHARED_HEALTH_CHECK
value: "true"
- name: REDIS_HOST
value: "redis-service"
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: redis-secret
key: password
```
## Migration
### From Independent Health Checks
1. **Enable Redis**: Ensure Redis is configured and accessible
2. **Enable Background Health Checks**: Set `background_health_checks: true`
3. **Enable Shared Health Check**: Set `use_shared_health_check: true`
4. **Deploy**: Update your proxy configuration
5. **Monitor**: Check `/health/shared-status` endpoint
### Rollback
To disable shared health check:
```yaml
general_settings:
use_shared_health_check: false
# background_health_checks can remain true for independent checks
```
## Related Features
- [Background Health Checks](./health.md#background-health-checks)
- [Redis Caching](./caching.md)
- [High Availability Setup](./db_deadlocks.md)
- [Health Check Endpoints](./health.md#health-endpoints)

61
docs/my-website/docs/proxy/sync_models_github.md Normal file
View file

@ -0,0 +1,61 @@
# Syncing Models to GitHub model_context_window
Sync model pricing data from GitHub's `model_prices_and_context_window.json` file outside of the LiteLLM UI.
> **📹 Video Tutorial**: [Watch how to sync models via the Admin UI](https://www.loom.com/share/ba41acc1882d41b284bbddbb0e9c27ce?sid=bdae351e-2026-4e39-932b-fcb185ff612c)
## Quick Start
**Manual sync:**
```bash
curl -X POST "https://your-proxy-url/reload/model_cost_map" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN" \
-H "Content-Type: application/json"
```
**Automatic sync every 6 hours:**
```bash
curl -X POST "https://your-proxy-url/schedule/model_cost_map_reload?hours=6" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN" \
-H "Content-Type: application/json"
```
## API Endpoints
| Endpoint | Method | Description |
|----------|--------|-------------|
| `/reload/model_cost_map` | POST | Manual sync |
| `/schedule/model_cost_map_reload?hours={hours}` | POST | Schedule periodic sync |
| `/schedule/model_cost_map_reload` | DELETE | Cancel scheduled sync |
| `/schedule/model_cost_map_reload/status` | GET | Check sync status |
**Authentication:** Requires admin role or master key
## Python Example
```python
import requests
def sync_models(proxy_url, admin_token):
response = requests.post(
f"{proxy_url}/reload/model_cost_map",
headers={"Authorization": f"Bearer {admin_token}"}
)
return response.json()
# Usage
result = sync_models("https://your-proxy-url", "your-admin-token")
print(result['message'])
```
## Configuration
**Custom model cost map URL:**
```bash
export LITELLM_MODEL_COST_MAP_URL="https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
```
**Use local model cost map:**
```bash
export LITELLM_LOCAL_MODEL_COST_MAP=True
```

277
docs/my-website/docs/proxy/tag_budgets.md Normal file
View file

@ -0,0 +1,277 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Setting Tag Budgets
Track spend and set budgets for your API requests using tags. Tags allow you to categorize and monitor costs across different cost centers, projects, and departments.
## Pre-Requisites
- You must set up a Postgres database (e.g. Supabase, Neon, etc.)
## What are Tags?
Tags are labels you can attach to your LLM requests to track and limit spending by category.
**Common Use Cases:**
- **Cost Center Tracking**: Allocate LLM costs to specific departments or business units (e.g., "engineering", "marketing", "customer-support")
- **Project-based Budgeting**: Set budgets for different projects or initiatives (e.g., "project-alpha", "chatbot-v2")
- **Customer Attribution**: Track spend per customer or client (e.g., "customer-acme", "customer-techcorp")
- **Feature Monitoring**: Monitor costs for specific features (e.g., "feature-chat", "feature-summarization")
Tags are added to each request in the `metadata` field to track and enforce budget limits.
## Setting Tag Budgets
### 1. Create a tag with budget
Create a tag to represent a cost center, project, or any budget category. Set `max_budget` ($ value allowed) and `budget_duration` (how frequently the budget resets).
**Example:** Create a tag for your Engineering department with a monthly $500 budget
#### API
Create a new tag and set `max_budget` and `budget_duration`
```shell
curl -X POST 'http://0.0.0.0:4000/tag/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"name": "engineering",
"description": "Engineering department cost center",
"max_budget": 500.0,
"budget_duration": "30d"
}'
```
**Request Body Parameters:**
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `name` | string | Yes | Unique name for the tag (e.g., cost center name) |
| `description` | string | No | Description of what this tag tracks |
| `models` | list[string] | No | Restrict tag to specific models |
| `max_budget` | float | No | Maximum budget in USD |
| `budget_duration` | string | No | How often budget resets (e.g., "30d", "1d") |
| `soft_budget` | float | No | Soft budget limit for warnings |
**Response:**
```json
{
"name": "engineering",
"description": "Engineering department cost center",
"max_budget": 500.0,
"budget_duration": "30d",
"budget_reset_at": "2025-11-10T00:00:00Z",
"created_at": "2025-10-11T00:00:00Z"
}
```
#### LiteLLM Admin UI
Navigate to the **Tag Management** page and click **Create New Tag**. Fill in the tag details and set your budget:
<Image
img={require('../../img/tag_budget1.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br />
**Possible values for `budget_duration`:**
| `budget_duration` | When Budget will reset |
| --- | --- |
| `budget_duration="1s"` | every 1 second |
| `budget_duration="1m"` | every 1 minute |
| `budget_duration="1h"` | every 1 hour |
| `budget_duration="1d"` | every 1 day |
| `budget_duration="7d"` | every 1 week |
| `budget_duration="30d"` | every 1 month |
### 2. Use the tag in your requests
Add tags to your API requests in the `metadata` field:
:::info Tags Budgets on API Keys
Currently, tag budget enforcement is only supported per request. If you'd like to set tags on API keys so all requests automatically inherit the tags budgets, please [create a feature request on GitHub](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeat%5D%3A).
:::
<Tabs>
<TabItem value="openai" label="OpenAI SDK">
```python
import openai
client = openai.OpenAI(
api_key="sk-1234", # Your LiteLLM proxy key
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="gpt-4",
messages=[{"role": "user", "content": "Hello"}],
extra_body={
"metadata": {
"tags": ["engineering"]
}
}
)
```
</TabItem>
<TabItem value="curl" label="cURL">
```shell
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
"metadata": {
"tags": ["engineering"]
}
}'
```
</TabItem>
</Tabs>
### 3. Test It
Make requests until the budget is exceeded:
```shell
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
"metadata": {
"tags": ["engineering"]
}
}'
```
**When budget is exceeded, you'll see:**
```json
{
"error": {
"message": "Budget has been exceeded! Tag=engineering Current cost: 505.50, Max budget: 500.0",
"type": "budget_exceeded",
"param": null,
"code": "400"
}
}
```
## Managing Tags
### View Tag Information
Get information about specific tags:
```shell
curl -X POST 'http://0.0.0.0:4000/tag/info' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"names": ["engineering", "marketing"]
}'
```
**Response:**
```json
{
"engineering": {
"name": "engineering",
"description": "Engineering department cost center",
"spend": 245.50,
"max_budget": 500.0,
"budget_duration": "30d",
"budget_reset_at": "2025-11-10T00:00:00Z",
"created_at": "2025-10-11T00:00:00Z",
"updated_at": "2025-10-11T12:30:00Z"
},
"marketing": {
"name": "marketing",
"description": "Marketing department cost center",
"spend": 89.20,
"max_budget": 300.0,
"budget_duration": "30d",
"budget_reset_at": "2025-11-10T00:00:00Z",
"created_at": "2025-10-11T00:00:00Z",
"updated_at": "2025-10-11T12:30:00Z"
}
}
```
### Update Tag Budget
Update an existing tag's budget:
```shell
curl -X POST 'http://0.0.0.0:4000/tag/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"name": "engineering",
"max_budget": 750.0,
"budget_duration": "30d"
}'
```
### Delete Tag
```shell
curl -X POST 'http://0.0.0.0:4000/tag/delete' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"name": "engineering"
}'
```
## Multiple Tags per Request
You can apply multiple tags to a single request to track costs across different dimensions simultaneously. For example, track both the cost center and the specific project:
```python
response = client.chat.completions.create(
model="gpt-4",
messages=[{"role": "user", "content": "Hello"}],
extra_body={
"metadata": {
"tags": ["engineering", "project-alpha", "customer-acme"]
}
}
)
```
```shell
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
"metadata": {
"tags": ["engineering", "project-alpha", "customer-acme"]
}
}'
```
**Budget Enforcement:** If any tag exceeds its budget, the request will be rejected.

14
docs/my-website/docs/proxy/ui.md
View file

@ -54,6 +54,20 @@ Allow others to create/delete their own keys.
[**Go Here**](./self_serve.md)
## Model Management
The Admin UI provides comprehensive model management capabilities:
- **Add Models**: Add new models through the UI without restarting the proxy
- **Model Hub**: Make models public for developers to discover available models
- **Price Data Sync**: Keep model pricing data up to date by syncing from GitHub
For detailed information on model management, see [Model Management](./model_management.md).
:::tip Sync Model Pricing Data
[Sync model pricing data from GitHub](./sync_models_github.md) to keep your model cost information current.
:::
## Disable Admin UI
Set `DISABLE_ADMIN_UI="True"` in your environment to disable the Admin UI.

133
docs/my-website/docs/proxy/virtual_keys.md
View file

@ -1,5 +1,6 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# Virtual Keys
Track Spend, and control model access via virtual keys for the proxy
@ -66,50 +67,6 @@ curl 'http://0.0.0.0:4000/key/generate' \
--data-raw '{"models": ["gpt-3.5-turbo", "gpt-4"], "metadata": {"user": "ishaan@berri.ai"}}'
```
## 🔁 Scheduled Key Rotations (NEW in v1.77.5)
LiteLLM can now rotate **virtual keys automatically** on a schedule you define.
### How it works
1. When creating a virtual key you set `rotation_schedule` – a [cron expression](https://crontab.guru/).
2. LiteLLM stores the schedule in the DB and runs a background job that regenerates the key at the specified time.
3. Existing key string is invalidated; a **notification webhook** (if configured) is sent with the new key value.
### Create a key with rotation
```bash
curl 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer <your-master-key>' \
-H 'Content-Type: application/json' \
-d '{
"models": ["gpt-4o"],
"rotation_schedule": "0 0 * * SUN", # rotate every Sunday at 00:00 UTC
"webhook_url": "https://example.com/key-rotated"
}'
```
### Enable globally via env
Set these env vars when starting the proxy:
| Variable | Description | Default |
|----------|-------------|---------|
| `LITELLM_KEY_ROTATION_ENABLED` | Enable the rotation worker | `false` |
| `LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS` | How often to scan for keys to rotate | `86400` |
### Webhook payload
```json
{
"event": "virtual_key.rotated",
"old_key_id": "sk-abc...",
"new_key": "sk-def...",
"rotation_time": "2025-10-05T00:00:00Z"
}
```
If no `webhook_url` is provided the new key value is returned in the response of the `/key/rotate` REST call instead.
## Spend Tracking
Get spend per:
@ -604,6 +561,94 @@ curl 'http://localhost:4000/key/sk-1234/regenerate' \
[**👉 API REFERENCE DOCS**](https://litellm-api.up.railway.app/#/key%20management/regenerate_key_fn_key__key__regenerate_post)
### Scheduled Key Rotations
LiteLLM can rotate **virtual keys automatically** based on time intervals you define.
#### Prerequisites
1. **Database connection required** - Key rotation requires a connected database to track rotation schedules
2. **Enable the rotation worker** - Set environment variable `LITELLM_KEY_ROTATION_ENABLED=true`
3. **Configure check interval** - Optionally set `LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS` (default: 86400 seconds / 24 hours)
#### How it works
1. When creating a virtual key, set `auto_rotate: true` and `rotation_interval` (duration string)
2. LiteLLM calculates the next rotation time as `now + rotation_interval` and stores it in the database
3. A background job periodically checks for keys where the rotation time has passed
4. When a key is due for rotation, LiteLLM automatically regenerates it and invalidates the old key string
5. The new rotation time is calculated and the cycle continues
#### Create a key with auto rotation
**API**
```bash
curl 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer <your-master-key>' \
-H 'Content-Type: application/json' \
-d '{
"models": ["gpt-4o"],
"auto_rotate": true,
"rotation_interval": "30d"
}'
```
**LiteLLM UI**
On the LiteLLM UI, Navigate to the Keys page and click on `Generate Key` > `Key Lifecycle` > `Enable Auto Rotation`
<Image
img={require('../../img/key_r.png')}
style={{width: '30%', display: 'block', margin: '0'}}
/>
**Valid rotation_interval formats:**
- `"30s"` - 30 seconds
- `"30m"` - 30 minutes
- `"30h"` - 30 hours
- `"30d"` - 30 days
- `"90d"` - 90 days
#### Update existing key to enable rotation
**API**
```bash
curl 'http://0.0.0.0:4000/key/update' \
-H 'Authorization: Bearer <your-master-key>' \
-H 'Content-Type: application/json' \
-d '{
"key": "sk-existing-key",
"auto_rotate": true,
"rotation_interval": "90d"
}'
```
**LiteLLM UI**
On the LiteLLM UI, Navigate to the Keys page. Select the key you want to update and click on `Edit Settings` > `Auto-Rotation Settings`
<Image
img={require('../../img/key_u.png')}
style={{width: '30%', display: 'block', margin: '0'}}
/>
#### Environment variables
Set these environment variables when starting the proxy:
| Variable | Description | Default |
|----------|-------------|---------|
| `LITELLM_KEY_ROTATION_ENABLED` | Enable the rotation worker | `false` |
| `LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS` | How often to scan for keys to rotate (in seconds) | `86400` (24 hours) |
**Example:**
```bash
export LITELLM_KEY_ROTATION_ENABLED=true
export LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS=3600 # Check every hour
litellm --config config.yaml
```
### Temporary Budget Increase
Use the `/key/update` endpoint to increase the budget of an existing key.

50
docs/my-website/docs/response_api.md
View file

@ -14,6 +14,7 @@ Requests to /chat/completions may be bridged here automatically when the provide
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Streaming | ✅ | |
| Image Generation Streaming | ✅ | Progressive image generation with partial images (1-3) |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Supported operations | Create a response, Get a response, Delete a response | |
@ -56,6 +57,29 @@ for event in response:
print(event)
```
#### Image Generation with Streaming
```python showLineNumbers title="OpenAI Streaming Image Generation"
import litellm
import base64
# Streaming image generation with partial images
stream = litellm.responses(
model="gpt-4.1", # Use an actual image generation model
input="Generate a gorgeous image of a river made of white owl feathers",
stream=True,
tools=[{"type": "image_generation", "partial_images": 2}],
)
for event in stream:
if event.type == "response.image_generation_call.partial_image":
idx = event.partial_image_index
image_base64 = event.partial_image_b64
image_bytes = base64.b64decode(image_base64)
with open(f"river{idx}.png", "wb") as f:
f.write(image_bytes)
```
#### GET a Response
```python showLineNumbers title="Get Response by ID"
import litellm
@ -380,6 +404,32 @@ for event in response:
print(event)
```
#### Image Generation with Streaming
```python showLineNumbers title="OpenAI Proxy Streaming Image Generation"
from openai import OpenAI
import base64
client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
stream = client.responses.create(
model="gpt-4.1",
input="Draw a gorgeous image of a river made of white owl feathers, snaking its way through a serene winter landscape",
stream=True,
tools=[{"type": "image_generation", "partial_images": 2}],
)
for event in stream:
print(f"event: {event}")
if event.type == "response.image_generation_call.partial_image":
idx = event.partial_image_index
image_base64 = event.partial_image_b64
image_bytes = base64.b64decode(image_base64)
with open(f"river{idx}.png", "wb") as f:
f.write(image_bytes)
```
#### GET a Response
```python showLineNumbers title="Get Response by ID with OpenAI SDK"
from openai import OpenAI

81
docs/my-website/docs/tutorials/claude_responses_api.md
View file

@ -105,7 +105,7 @@ LITELLM_MASTER_KEY gives claude access to all proxy models, whereas a virtual ke
Alternatively, use the Anthropic pass-through endpoint:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
@ -209,4 +209,81 @@ claude --model claude-bedrock
</TabItem>
</Tabs>
<Image img={require('../../img/release_notes/claude_code_demo.png')} style={{ width: '500px', height: 'auto' }} />
<Image img={require('../../img/release_notes/claude_code_demo.png')} style={{ width: '500px', height: 'auto' }} />
## Connecting MCP Servers
You can also connect MCP servers to Claude Code via LiteLLM Proxy.
:::note
Limitations:
- Currently, only HTTP MCP servers are supported
- Does not work in Cursor IDE yet.
:::
1. Add the MCP server to your `config.yaml`
In this example, we'll add the Github MCP server to our `config.yaml`
```yaml title="config.yaml" showLineNumbers
mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
auth_type: oauth2
authorization_url: https://github.com/login/oauth/authorize
token_url: https://github.com/login/oauth/access_token
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
scopes: ["public_repo", "user:email"]
```
2. Start LiteLLM Proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Use the MCP server in Claude Code
```bash
claude mcp add --transport http litellm_proxy http://0.0.0.0:4000/github_mcp/mcp --header "Authorization: Bearer sk-LITELLM_VIRTUAL_KEY"
```
4. Authenticate via Claude Code
a. Start Claude Code
```bash
claude
```
b. Authenticate via Claude Code
```bash
/mcp
```
c. Select the MCP server
```bash
> litellm_proxy
```
d. Start Oauth flow via Claude Code
```bash
> 1. Authenticate
2. Reconnect
3. Disable
```
e. Once completed, you should see this success message:
<Image img={require('../../img/oauth_2_success.png')} style={{ width: '500px', height: 'auto' }} />

48
docs/my-website/docs/tutorials/msft_sso.md
View file

@ -140,6 +140,54 @@ litellm_settings:
<Image img={require('../../img/msft_default_settings.png')} style={{ width: '900px', height: 'auto' }} />
## 4. Using Entra ID App Roles for User Permissions
You can assign user roles directly from Entra ID using App Roles. LiteLLM will automatically read the app roles from the JWT token during SSO sign-in and assign the corresponding role to the user.
### 4.1 Supported Roles
LiteLLM supports the following app roles (case-insensitive):
- `proxy_admin` - Admin over the entire LiteLLM platform
- `proxy_admin_viewer` - Read-only admin access (can view all keys and spend)
- `org_admin` - Admin over a specific organization (can create teams and users within their org)
- `internal_user` - Standard user (can create/view/delete their own keys and view their own spend)
### 4.2 Create App Roles in Entra ID
1. Navigate to your App Registration on https://portal.azure.com/
2. Go to **App roles** > **Create app role**
3. Configure the app role:
- **Display name**: Proxy Admin (or your preferred display name)
- **Value**: `proxy_admin` (use one of the supported role values above)
- **Description**: Administrator access to LiteLLM proxy
- **Allowed member types**: Users/Groups
4. Click **Apply** to save the role
### 4.3 Assign Users to App Roles
1. Navigate to **Enterprise Applications** on https://portal.azure.com/
2. Select your LiteLLM application
3. Go to **Users and groups** > **Add user/group**
4. Select the user and assign them to one of the app roles you created
### 4.4 Test the Role Assignment
1. Sign in to LiteLLM UI via SSO as a user with an assigned app role
2. LiteLLM will automatically extract the app role from the JWT token
3. The user will be assigned the corresponding LiteLLM role in the database
4. The user's permissions will reflect their assigned role
**How it works:**
- When a user signs in via Microsoft SSO, LiteLLM extracts the `roles` claim from the JWT `id_token`
- If any of the roles match a valid LiteLLM role (case-insensitive), that role is assigned to the user
- If multiple roles are present, LiteLLM uses the first valid role it finds
- This role assignment persists in the LiteLLM database and determines the user's access level
## Video Walkthrough
This walks through setting up sso auto-add for **Microsoft Entra ID**

BIN
docs/my-website/img/key_r.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 125 KiB

BIN
docs/my-website/img/key_u.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 216 KiB

BIN
docs/my-website/img/mcp_updates.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 913 KiB

BIN
docs/my-website/img/oauth_2_success.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 67 KiB

BIN
docs/my-website/img/release_notes/1_78_0_perf.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 134 KiB

BIN
docs/my-website/img/release_notes/perf_77_5.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.3 MiB

BIN
docs/my-website/img/release_notes/perf_77_7.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 253 KiB

BIN
docs/my-website/img/release_notes/schedule_key_rotations.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 603 KiB

BIN
docs/my-website/img/release_notes/tool_control.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 798 KiB

BIN
docs/my-website/img/tag_budget1.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 305 KiB

BIN
docs/my-website/img/tag_budget2.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 416 KiB

1
docs/my-website/release_notes/v1.75.5-stable/index.md
View file

@ -50,6 +50,7 @@ pip install litellm==1.75.5.post2
- **Oracle Cloud Infrastructure** - New LLM provider for calling models on Oracle Cloud Infrastructure.
- **Digital Ocean's Gradient AI** - New LLM provider for calling models on Digital Ocean's Gradient AI platform.
---
### Risk of Upgrade

44
docs/my-website/release_notes/v1.77.5-stable/index.md
View file

@ -1,5 +1,5 @@
---
title: "[Preview] v1.77.5-stable - MCP OAuth 2.0 Support"
title: "v1.77.5-stable - MCP OAuth 2.0 Support"
slug: "v1-77-5"
date: 2025-09-29T10:00:00
authors:
@ -11,6 +11,10 @@ authors:
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Alexsander Hamir
title: Backend Performance Engineer
url: https://www.linkedin.com/in/alexsander-baptista/
image_url: https://media.licdn.com/dms/image/v2/D5603AQGXnziu4kqNCQ/profile-displayphoto-crop_800_800/B56ZkxEcuOKEAI-/0/1757464874550?e=1762387200&v=beta&t=9SNXLsWhx8OnYPAMQ9fqAr02oevDYEAL2vMYg2f9ieg
hide_table_of_contents: false
---
@ -28,7 +32,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.77.5.rc.1
ghcr.io/berriai/litellm:v1.77.5-stable
```
</TabItem>
@ -49,7 +53,41 @@ pip install litellm==1.77.5
- **MCP OAuth 2.0 Support** - Enhanced authentication for Model Context Protocol integrations
- **Scheduled Key Rotations** - Automated key rotation capabilities for enhanced security
- **New Gemini 2.5 Flash & Flash-lite Models** - Latest September 2025 preview models with improved pricing and features
- **Performance Improvements** - Critical InMemoryCache unbounded growth resolution
- **Performance Improvements** - 54% RPS improvement
---
### Performance Improvements - 54% RPS Improvement
<Image img={require('../../img/release_notes/perf_77_5.png')} style={{ width: '800px', height: 'auto' }} />
<br/>
This release brings a 54% RPS improvement (1,040 → 1,602 RPS, aggregated) per instance.
The improvement comes from fixing O(n²) inefficiencies in the LiteLLM Router, primarily caused by repeated use of `in` statements inside loops over large arrays.
Tests were run with a database-only setup (no cache hits).
#### Test Setup
All benchmarks were executed using Locust with 1,000 concurrent users and a ramp-up of 500. The environment was configured to stress the routing layer and eliminate caching as a variable.
**System Specs**
- **CPU:** 8 vCPUs
- **Memory:** 32 GB RAM
**Configuration (config.yaml)**
View the complete configuration: [gist.github.com/AlexsanderHamir/config.yaml](https://gist.github.com/AlexsanderHamir/53f7d554a5d2afcf2c4edb5b6be68ff4)
**Load Script (no_cache_hits.py)**
View the complete load testing script: [gist.github.com/AlexsanderHamir/no_cache_hits.py](https://gist.github.com/AlexsanderHamir/42c33d7a4dc7a57f56a78b560dee3a42)
---
## New Models / Updated Models

389
docs/my-website/release_notes/v1.77.7-stable/index.md Normal file
View file

@ -0,0 +1,389 @@
---
title: "v1.77.7-stable - 2.9x Lower Median Latency"
slug: "v1-77-7"
date: 2025-10-04T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Alexsander Hamir
title: Backend Performance Engineer
url: https://www.linkedin.com/in/alexsander-baptista/
image_url: https://media.licdn.com/dms/image/v2/D5603AQGXnziu4kqNCQ/profile-displayphoto-crop_800_800/B56ZkxEcuOKEAI-/0/1757464874550?e=1762387200&v=beta&t=9SNXLsWhx8OnYPAMQ9fqAr02oevDYEAL2vMYg2f9ieg
- name: Achintya Rajan
title: Fullstack Engineer
url: https://www.linkedin.com/in/achintya-rajan/
image_url: https://media.licdn.com/dms/image/v2/D5603AQGdkEeyJTdljw/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1716271140869?e=1762387200&v=beta&t=9gOoLPeqR2E5z3KSX61EUj3HVZXmgo87vhVuSHeffjc
- name: Sameer Kankute
title: Backend Engineer (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1762387200&v=beta&t=0jbuX-f4eSnDxBY3olI6meuYr-LMbObhFmFbRcKF5mY
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.77.7.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.77.7.rc.1
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Dynamic Rate Limiter v3** - Automatically maximizes throughput when capacity is available (< 80% saturation) by allowing lower-priority requests to use unused capacity, then switches to fair priority-based allocation under high load (≥ 80%) to prevent blocking
- **Major Performance Improvements** - 2.9x lower median latency at 1,000 concurrent users.
- **Claude Sonnet 4.5** - Support for Anthropic's new Claude Sonnet 4.5 model family with 200K+ context and tiered pricing
- **MCP Gateway Enhancements** - Fine-grained tool control, server permissions, and forwardable headers
- **AMD Lemonade & Nvidia NIM** - New provider support for AMD Lemonade and Nvidia NIM Rerank
- **GitLab Prompt Management** - GitLab-based prompt management integration
### Performance - 2.9x Lower Median Latency
<Image img={require('../../img/release_notes/perf_77_7.png')} style={{ width: '800px', height: 'auto' }} />
<br/>
This update removes LiteLLM router inefficiencies, reducing complexity from O(M×N) to O(1). Previously, it built a new array and ran repeated checks like data["model"] in llm_router.get_model_ids(). Now, a direct ID-to-deployment map eliminates redundant allocations and scans.
As a result, performance improved across all latency percentiles:
- **Median latency:** 320 ms → **110 ms** (−65.6%)
- **p95 latency:** 850 ms → **440 ms** (−48.2%)
- **p99 latency:** 1,400 ms → **810 ms** (−42.1%)
- **Average latency:** 864 ms → **310 ms** (−64%)
#### Test Setup
**Locust**
- **Concurrent users:** 1,000
- **Ramp-up:** 500
**System Specs**
- **CPU:** 4 vCPUs
- **Memory:** 8 GB RAM
- **LiteLLM Workers:** 4
- **Instances**: 4
**Configuration (config.yaml)**
View the complete configuration: [gist.github.com/AlexsanderHamir/config.yaml](https://gist.github.com/AlexsanderHamir/53f7d554a5d2afcf2c4edb5b6be68ff4)
**Load Script (no_cache_hits.py)**
View the complete load testing script: [gist.github.com/AlexsanderHamir/no_cache_hits.py](https://gist.github.com/AlexsanderHamir/42c33d7a4dc7a57f56a78b560dee3a42)
### MCP OAuth 2.0 Support
<Image img={require('../../img/mcp_updates.jpg')} style={{ width: '800px', height: 'auto' }} />
<br/>
This release adds support for OAuth 2.0 Client Credentials for MCP servers. This is great for **Internal Dev Tools** use-cases, as it enables your users to call MCP servers, with their own credentials. E.g. Allowing your developers to call the Github MCP, with their own credentials.
[Set it up today on Claude Code](../../docs/tutorials/claude_responses_api#connecting-mcp-servers)
### Scheduled Key Rotations
<Image img={require('../../img/release_notes/schedule_key_rotations.png')} style={{ width: '800px', height: 'auto' }} />
<br/>
This release brings support for scheduling virtual key rotations on LiteLLM AI Gateway.
From this release you can enforce Virtual Keys to rotate on a schedule of your choice e.g every 15 days/30 days/60 days etc.
This is great for Proxy Admins who need to enforce security policies for production workloads.
[Get Started](../../docs/proxy/virtual_keys#scheduled-key-rotations)
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| Anthropic | `claude-sonnet-4-5` | 200K | $3.00 | $15.00 | Chat, reasoning, vision, function calling, prompt caching |
| Anthropic | `claude-sonnet-4-5-20250929` | 200K | $3.00 | $15.00 | Chat, reasoning, vision, function calling, prompt caching |
| Bedrock | `eu.anthropic.claude-sonnet-4-5-20250929-v1:0` | 200K | $3.00 | $15.00 | Chat, reasoning, vision, function calling, prompt caching |
| Azure AI | `azure_ai/grok-4` | 131K | $5.50 | $27.50 | Chat, reasoning, function calling, web search |
| Azure AI | `azure_ai/grok-4-fast-reasoning` | 131K | $0.43 | $1.73 | Chat, reasoning, function calling, web search |
| Azure AI | `azure_ai/grok-4-fast-non-reasoning` | 131K | $0.43 | $1.73 | Chat, function calling, web search |
| Azure AI | `azure_ai/grok-code-fast-1` | 131K | $3.50 | $17.50 | Chat, function calling, web search |
| Groq | `groq/moonshotai/kimi-k2-instruct-0905` | Context varies | Pricing varies | Pricing varies | Chat, function calling |
| Ollama | Ollama Cloud models | Varies | Free | Free | Self-hosted models via Ollama Cloud |
#### Features
- **[Anthropic](../../docs/providers/anthropic)**
- Add new claude-sonnet-4-5 model family with tiered pricing above 200K tokens - [PR #15041](https://github.com/BerriAI/litellm/pull/15041)
- Add anthropic/claude-sonnet-4-5 to model price json with prompt caching support - [PR #15049](https://github.com/BerriAI/litellm/pull/15049)
- Add 200K prices for Sonnet 4.5 - [PR #15140](https://github.com/BerriAI/litellm/pull/15140)
- Add cost tracking for /v1/messages in streaming response - [PR #15102](https://github.com/BerriAI/litellm/pull/15102)
- Add /v1/messages/count_tokens to Anthropic routes for non-admin user access - [PR #15034](https://github.com/BerriAI/litellm/pull/15034)
- **[Gemini](../../docs/providers/gemini)**
- Ignore type param for gemini tools - [PR #15022](https://github.com/BerriAI/litellm/pull/15022)
- **[Vertex AI](../../docs/providers/vertex)**
- Add LiteLLM Overhead metric for VertexAI - [PR #15040](https://github.com/BerriAI/litellm/pull/15040)
- Support googlemap grounding in vertex ai - [PR #15179](https://github.com/BerriAI/litellm/pull/15179)
- **[Azure](../../docs/providers/azure)**
- Add azure_ai grok-4 model family - [PR #15137](https://github.com/BerriAI/litellm/pull/15137)
- Use the `extra_query` parameter for GET requests in Azure Batch - [PR #14997](https://github.com/BerriAI/litellm/pull/14997)
- Use extra_query for download results (Batch API) - [PR #15025](https://github.com/BerriAI/litellm/pull/15025)
- Add support for Azure AD token-based authorization - [PR #14813](https://github.com/BerriAI/litellm/pull/14813)
- **[Ollama](../../docs/providers/ollama)**
- Add ollama cloud models - [PR #15008](https://github.com/BerriAI/litellm/pull/15008)
- **[Groq](../../docs/providers/groq)**
- Add groq/moonshotai/kimi-k2-instruct-0905 - [PR #15079](https://github.com/BerriAI/litellm/pull/15079)
- **[OpenAI](../../docs/providers/openai)**
- Add support for GPT 5 codex models - [PR #14841](https://github.com/BerriAI/litellm/pull/14841)
- **[DeepInfra](../../docs/providers/deepinfra)**
- Update DeepInfra model data refresh with latest pricing - [PR #14939](https://github.com/BerriAI/litellm/pull/14939)
- **[Bedrock](../../docs/providers/bedrock)**
- Add JP Cross-Region Inference - [PR #15188](https://github.com/BerriAI/litellm/pull/15188)
- Add "eu.anthropic.claude-sonnet-4-5-20250929-v1:0" - [PR #15181](https://github.com/BerriAI/litellm/pull/15181)
- Add twelvelabs bedrock Async Invoke Support - [PR #14871](https://github.com/BerriAI/litellm/pull/14871)
- **[Nvidia NIM](../../docs/providers/nvidia_nim)**
- Add Nvidia NIM Rerank Support - [PR #15152](https://github.com/BerriAI/litellm/pull/15152)
### Bug Fixes
- **[VLLM](../../docs/providers/vllm)**
- Fix response_format bug in hosted vllm audio_transcription - [PR #15010](https://github.com/BerriAI/litellm/pull/15010)
- Fix passthrough of atranscription into kwargs going to upstream provider - [PR #15005](https://github.com/BerriAI/litellm/pull/15005)
- **[OCI](../../docs/providers/oci)**
- Fix OCI Generative AI Integration when using Proxy - [PR #15072](https://github.com/BerriAI/litellm/pull/15072)
- **General**
- Fix: Authorization header to use correct "Bearer" capitalization - [PR #14764](https://github.com/BerriAI/litellm/pull/14764)
- Bug fix: gpt-5-chat-latest has incorrect max_input_tokens value - [PR #15116](https://github.com/BerriAI/litellm/pull/15116)
- Update request handling for original exceptions - [PR #15013](https://github.com/BerriAI/litellm/pull/15013)
#### New Provider Support
- **[AMD Lemonade](../../docs/providers/lemonade)**
- Add AMD Lemonade provider support - [PR #14840](https://github.com/BerriAI/litellm/pull/14840)
---
## LLM API Endpoints
#### Features
- **[Responses API](../../docs/response_api)**
- Return Cost for Responses API Streaming requests - [PR #15053](https://github.com/BerriAI/litellm/pull/15053)
- **[/generateContent](../../docs/providers/gemini)**
- Add full support for native Gemini API translation - [PR #15029](https://github.com/BerriAI/litellm/pull/15029)
- **Passthrough Gemini Routes**
- Add Gemini generateContent passthrough cost tracking - [PR #15014](https://github.com/BerriAI/litellm/pull/15014)
- Add streamGenerateContent cost tracking in passthrough - [PR #15199](https://github.com/BerriAI/litellm/pull/15199)
- **Passthrough Vertex AI Routes**
- Add cost tracking for Vertex AI Passthrough `/predict` endpoint - [PR #15019](https://github.com/BerriAI/litellm/pull/15019)
- Add cost tracking for Vertex AI Live API WebSocket Passthrough - [PR #14956](https://github.com/BerriAI/litellm/pull/14956)
- **General**
- Preserve Whitespace Characters in Model Response Streams - [PR #15160](https://github.com/BerriAI/litellm/pull/15160)
- Add provider name to payload specification - [PR #15130](https://github.com/BerriAI/litellm/pull/15130)
- Ensure query params are forwarded from origin url to downstream request - [PR #15087](https://github.com/BerriAI/litellm/pull/15087)
---
## Management Endpoints / UI
#### Features
- **Virtual Keys**
- Ensure LLM_API_KEYs can access pass through routes - [PR #15115](https://github.com/BerriAI/litellm/pull/15115)
- Support 'guaranteed_throughput' when setting limits on keys belonging to a team - [PR #15120](https://github.com/BerriAI/litellm/pull/15120)
- **Models + Endpoints**
- Ensure OCI secret fields not shared on /models and /v1/models endpoints - [PR #15085](https://github.com/BerriAI/litellm/pull/15085)
- Add snowflake on UI - [PR #15083](https://github.com/BerriAI/litellm/pull/15083)
- Make UI theme settings publicly accessible for custom branding - [PR #15074](https://github.com/BerriAI/litellm/pull/15074)
- **Admin Settings**
- Ensure OTEL settings are saved in DB after set on UI - [PR #15118](https://github.com/BerriAI/litellm/pull/15118)
- Top api key tags - [PR #15151](https://github.com/BerriAI/litellm/pull/15151), [PR #15156](https://github.com/BerriAI/litellm/pull/15156)
- **MCP**
- show health status of MCP servers - [PR #15185](https://github.com/BerriAI/litellm/pull/15185)
- allow setting extra headers on the UI - [PR #15185](https://github.com/BerriAI/litellm/pull/15185)
- allow editing allowed tools on the UI - [PR #15185](https://github.com/BerriAI/litellm/pull/15185)
### Bug Fixes
- **Virtual Keys**
- (security) prevent user key from updating other user keys - [PR #15201](https://github.com/BerriAI/litellm/pull/15201)
- (security) don't return all keys with blank key alias on /v2/key/info - [PR #15201](https://github.com/BerriAI/litellm/pull/15201)
- Fix Session Token Cookie Infinite Logout Loop - [PR #15146](https://github.com/BerriAI/litellm/pull/15146)
- **Models + Endpoints**
- Make UI theme settings publicly accessible for custom branding - [PR #15074](https://github.com/BerriAI/litellm/pull/15074)
- **Teams**
- fix failed copy to clipboard for http ui - [PR #15195](https://github.com/BerriAI/litellm/pull/15195)
- **Logs**
- fix logs page render logs on filter lookup - [PR #15195](https://github.com/BerriAI/litellm/pull/15195)
- fix lookup list of end users (migrate to more efficient /customers/list lookup) - [PR #15195](https://github.com/BerriAI/litellm/pull/15195)
- **Test key**
- update selected model on key change - [PR #15197](https://github.com/BerriAI/litellm/pull/15197)
- **Dashboard**
- Fix LiteLLM model name fallback in dashboard overview - [PR #14998](https://github.com/BerriAI/litellm/pull/14998)
---
## Logging / Guardrail / Prompt Management Integrations
#### Features
- **[OpenTelemetry](../../docs/observability/otel)**
- Use generation_name for span naming in logging method - [PR #14799](https://github.com/BerriAI/litellm/pull/14799)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Handle non-serializable objects in Langfuse logging - [PR #15148](https://github.com/BerriAI/litellm/pull/15148)
- Set usage_details.total in langfuse integration - [PR #15015](https://github.com/BerriAI/litellm/pull/15015)
- **[Prometheus](../../docs/proxy/prometheus)**
- support custom metadata labels on key/team - [PR #15094](https://github.com/BerriAI/litellm/pull/15094)
#### Guardrails
- **[Javelin](../../docs/proxy/guardrails)**
- Add Javelin standalone guardrails integration for LiteLLM Proxy - [PR #14983](https://github.com/BerriAI/litellm/pull/14983)
- Add logging for important status fields in guardrails - [PR #15090](https://github.com/BerriAI/litellm/pull/15090)
- Don't run post_call guardrail if no text returned from Bedrock - [PR #15106](https://github.com/BerriAI/litellm/pull/15106)
#### Prompt Management
- **[GitLab](../../docs/proxy/prompt_management)**
- GitLab based Prompt manager - [PR #14988](https://github.com/BerriAI/litellm/pull/14988)
---
## Spend Tracking, Budgets and Rate Limiting
- **Cost Tracking**
- Proxy: end user cost tracking in the responses API - [PR #15124](https://github.com/BerriAI/litellm/pull/15124)
- **Parallel Request Limiter v3**
- Use well known redis cluster hashing algorithm - [PR #15052](https://github.com/BerriAI/litellm/pull/15052)
- Fixes to dynamic rate limiter v3 - add saturation detection - [PR #15119](https://github.com/BerriAI/litellm/pull/15119)
- Dynamic Rate Limiter v3 - fixes for detecting saturation + fixes for post saturation behavior - [PR #15192](https://github.com/BerriAI/litellm/pull/15192)
- **Teams**
- Add model specific tpm/rpm limits to teams on LiteLLM - [PR #15044](https://github.com/BerriAI/litellm/pull/15044)
---
## MCP Gateway
- **Server Configuration**
- Specify forwardable headers, specify allowed/disallowed tools for MCP servers - [PR #15002](https://github.com/BerriAI/litellm/pull/15002)
- Enforce server permissions on call tools - [PR #15044](https://github.com/BerriAI/litellm/pull/15044)
- MCP Gateway Fine-grained Tools Addition - [PR #15153](https://github.com/BerriAI/litellm/pull/15153)
- **Bug Fixes**
- Remove servername prefix mcp tools tests - [PR #14986](https://github.com/BerriAI/litellm/pull/14986)
- Resolve regression with duplicate Mcp-Protocol-Version header - [PR #15050](https://github.com/BerriAI/litellm/pull/15050)
- Fix test_mcp_server.py - [PR #15183](https://github.com/BerriAI/litellm/pull/15183)
---
## Performance / Loadbalancing / Reliability improvements
- **Router Optimizations**
- **+62.5% P99 Latency Improvement** - Remove router inefficiencies (from O(M*N) to O(1)) - [PR #15046](https://github.com/BerriAI/litellm/pull/15046)
- Remove hasattr checks in Router - [PR #15082](https://github.com/BerriAI/litellm/pull/15082)
- Remove Double Lookups - [PR #15084](https://github.com/BerriAI/litellm/pull/15084)
- Optimize _filter_cooldown_deployments from O(n×m + k×n) to O(n) - [PR #15091](https://github.com/BerriAI/litellm/pull/15091)
- Optimize unhealthy deployment filtering in retry path (O(n*m) → O(n+m)) - [PR #15110](https://github.com/BerriAI/litellm/pull/15110)
- **Cache Optimizations**
- Reduce complexity of InMemoryCache.evict_cache from O(n*log(n)) to O(log(n)) - [PR #15000](https://github.com/BerriAI/litellm/pull/15000)
- Avoiding expensive operations when cache isn't available - [PR #15182](https://github.com/BerriAI/litellm/pull/15182)
- **Worker Management**
- Add proxy CLI option to recycle workers after N requests - [PR #15007](https://github.com/BerriAI/litellm/pull/15007)
- **Metrics & Monitoring**
- LiteLLM Overhead metric tracking - Add support for tracking litellm overhead on cache hits - [PR #15045](https://github.com/BerriAI/litellm/pull/15045)
---
## Documentation Updates
- **Provider Documentation**
- Update litellm docs from latest release - [PR #15004](https://github.com/BerriAI/litellm/pull/15004)
- Add missing api_key parameter - [PR #15058](https://github.com/BerriAI/litellm/pull/15058)
- **General Documentation**
- Use docker compose instead of docker-compose - [PR #15024](https://github.com/BerriAI/litellm/pull/15024)
- Add railtracks to projects that are using litellm - [PR #15144](https://github.com/BerriAI/litellm/pull/15144)
- Perf: Last week improvement - [PR #15193](https://github.com/BerriAI/litellm/pull/15193)
- Sync models GitHub documentation with Loom video and cross-reference - [PR #15191](https://github.com/BerriAI/litellm/pull/15191)
---
## Security Fixes
- **JWT Token Security** - Don't log JWT SSO token on .info() log - [PR #15145](https://github.com/BerriAI/litellm/pull/15145)
---
## New Contributors
* @herve-ves made their first contribution in [PR #14998](https://github.com/BerriAI/litellm/pull/14998)
* @wenxi-onyx made their first contribution in [PR #15008](https://github.com/BerriAI/litellm/pull/15008)
* @jpetrucciani made their first contribution in [PR #15005](https://github.com/BerriAI/litellm/pull/15005)
* @abhijitjavelin made their first contribution in [PR #14983](https://github.com/BerriAI/litellm/pull/14983)
* @ZeroClover made their first contribution in [PR #15039](https://github.com/BerriAI/litellm/pull/15039)
* @cedarm made their first contribution in [PR #15043](https://github.com/BerriAI/litellm/pull/15043)
* @Isydmr made their first contribution in [PR #15025](https://github.com/BerriAI/litellm/pull/15025)
* @serializer made their first contribution in [PR #15013](https://github.com/BerriAI/litellm/pull/15013)
* @eddierichter-amd made their first contribution in [PR #14840](https://github.com/BerriAI/litellm/pull/14840)
* @malags made their first contribution in [PR #15000](https://github.com/BerriAI/litellm/pull/15000)
* @henryhwang made their first contribution in [PR #15029](https://github.com/BerriAI/litellm/pull/15029)
* @plafleur made their first contribution in [PR #15111](https://github.com/BerriAI/litellm/pull/15111)
* @tyler-liner made their first contribution in [PR #14799](https://github.com/BerriAI/litellm/pull/14799)
* @Amir-R25 made their first contribution in [PR #15144](https://github.com/BerriAI/litellm/pull/15144)
* @georg-wolflein made their first contribution in [PR #15124](https://github.com/BerriAI/litellm/pull/15124)
* @niharm made their first contribution in [PR #15140](https://github.com/BerriAI/litellm/pull/15140)
* @anthony-liner made their first contribution in [PR #15015](https://github.com/BerriAI/litellm/pull/15015)
* @rishiganesh2002 made their first contribution in [PR #15153](https://github.com/BerriAI/litellm/pull/15153)
* @danielaskdd made their first contribution in [PR #15160](https://github.com/BerriAI/litellm/pull/15160)
* @JVenberg made their first contribution in [PR #15146](https://github.com/BerriAI/litellm/pull/15146)
* @speglich made their first contribution in [PR #15072](https://github.com/BerriAI/litellm/pull/15072)
* @daily-kim made their first contribution in [PR #14764](https://github.com/BerriAI/litellm/pull/14764)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.77.5.rc.4...v1.77.7.rc.1)**

394
docs/my-website/release_notes/v1.78.0-stable/index.md Normal file
View file

@ -0,0 +1,394 @@
---
title: "[Preview] v1.78.0-stable - MCP Gateway: Control Tool Access by Team, Key"
slug: "v1-78-0"
date: 2025-10-11T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Alexsander Hamir
title: Backend Performance Engineer
url: https://www.linkedin.com/in/alexsander-baptista/
image_url: https://media.licdn.com/dms/image/v2/D5603AQGXnziu4kqNCQ/profile-displayphoto-crop_800_800/B56ZkxEcuOKEAI-/0/1757464874550?e=1762387200&v=beta&t=9SNXLsWhx8OnYPAMQ9fqAr02oevDYEAL2vMYg2f9ieg
- name: Achintya Rajan
title: Fullstack Engineer
url: https://www.linkedin.com/in/achintya-rajan/
image_url: https://media.licdn.com/dms/image/v2/D5603AQGdkEeyJTdljw/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1716271140869?e=1762387200&v=beta&t=9gOoLPeqR2E5z3KSX61EUj3HVZXmgo87vhVuSHeffjc
- name: Sameer Kankute
title: Backend Engineer (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1762387200&v=beta&t=0jbuX-f4eSnDxBY3olI6meuYr-LMbObhFmFbRcKF5mY
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.78.0.rc.2
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.78.0.rc.2
```
</TabItem>
</Tabs>
---
## Key Highlights
- **MCP Gateway - Control Tool Access by Team, Key** - Control MCP tool access by team/key.
- **Performance Improvements** - 70% Lower p99 Latency
- **GPT-5 Pro & GPT-Image-1-Mini** - Day 0 support for OpenAI's GPT-5 Pro (400K context) and gpt-image-1-mini image generation
- **EnkryptAI Guardrails** - New guardrail integration for content moderation
- **Tag-Based Budgets** - Support for setting budgets based on request tags
---
### MCP Gateway - Control Tool Access by Team, Key
<Image
img={require('../../img/release_notes/tool_control.png')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<br/>
Proxy admins can now control MCP tool access by team or key. This makes it easy to grant different teams selective access to tools from the same MCP server.
For example, you can now give your Engineering team access to `list_repositories`, `create_issue`, and `search_code` tools, while Sales only gets `search_code` and `close_issue` tools.
This makes it easier for Proxy Admins to govern MCP Tool Access.
[Get Started](../../docs/mcp_control#set-allowed-tools-for-a-key-team-or-organization)
---
## Performance - 70% Lower p99 Latency
<Image img={require('../../img/release_notes/1_78_0_perf.png')} style={{ width: '800px', height: 'auto' }} />
<br/>
This release cuts p99 latency by 70% on LiteLLM AI Gateway, making it even better for low-latency use cases.
These gains come from two key enhancements:
**Reliable Sessions**
Added support for shared sessions with aiohttp. The shared_session parameter is now consistently used across all calls, enabling connection pooling.
**Faster Routing**
A new `model_name_to_deployment_indices` hash map replaces O(n) list scans in `_get_all_deployments()` with O(1) hash lookups, boosting routing performance and scalability.
As a result, performance improved across all latency percentiles:
- **Median latency:** 110 ms → **100 ms** (−9.1%)
- **p95 latency:** 440 ms → **150 ms** (−65.9%)
- **p99 latency:** 810 ms → **240 ms** (−70.4%)
- **Average latency:** 310 ms → **111.73 ms** (−64.0%)
### **Test Setup**
**Locust**
- **Concurrent users:** 1,000
- **Ramp-up:** 500
**System Specs**
- **Database was used**
- **CPU:** 4 vCPUs
- **Memory:** 8 GB RAM
- **LiteLLM Workers:** 4
- **Instances**: 4
**Configuration (config.yaml)**
View the complete configuration: [gist.github.com/AlexsanderHamir/config.yaml](https://gist.github.com/AlexsanderHamir/53f7d554a5d2afcf2c4edb5b6be68ff4)
**Load Script (no_cache_hits.py)**
View the complete load testing script: [gist.github.com/AlexsanderHamir/no_cache_hits.py](https://gist.github.com/AlexsanderHamir/42c33d7a4dc7a57f56a78b560dee3a42)
---
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| OpenAI | `gpt-5-pro` | 400K | $15.00 | $120.00 | Responses API, reasoning, vision, function calling, prompt caching, web search |
| OpenAI | `gpt-5-pro-2025-10-06` | 400K | $15.00 | $120.00 | Responses API, reasoning, vision, function calling, prompt caching, web search |
| OpenAI | `gpt-image-1-mini` | - | $2.00/img | - | Image generation and editing |
| OpenAI | `gpt-realtime-mini` | 128K | $0.60 | $2.40 | Realtime audio, function calling |
| Azure AI | `azure_ai/Phi-4-mini-reasoning` | 131K | $0.08 | $0.32 | Function calling |
| Azure AI | `azure_ai/Phi-4-reasoning` | 32K | $0.125 | $0.50 | Function calling, reasoning |
| Azure AI | `azure_ai/MAI-DS-R1` | 128K | $1.35 | $5.40 | Reasoning, function calling |
| Bedrock | `au.anthropic.claude-sonnet-4-5-20250929-v1:0` | 200K | $3.30 | $16.50 | Chat, reasoning, vision, function calling, prompt caching |
| Bedrock | `global.anthropic.claude-sonnet-4-5-20250929-v1:0` | 200K | $3.00 | $15.00 | Chat, reasoning, vision, function calling, prompt caching |
| Bedrock | `global.anthropic.claude-sonnet-4-20250514-v1:0` | 1M | $3.00 | $15.00 | Chat, reasoning, vision, function calling, prompt caching |
| Bedrock | `cohere.embed-v4:0` | 128K | $0.12 | - | Embeddings, image input support |
| OCI | `oci/cohere.command-latest` | 128K | $1.56 | $1.56 | Function calling |
| OCI | `oci/cohere.command-a-03-2025` | 256K | $1.56 | $1.56 | Function calling |
| OCI | `oci/cohere.command-plus-latest` | 128K | $1.56 | $1.56 | Function calling |
| Together AI | `together_ai/moonshotai/Kimi-K2-Instruct-0905` | 262K | $1.00 | $3.00 | Function calling |
| Together AI | `together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct` | 262K | $0.15 | $1.50 | Function calling |
| Together AI | `together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking` | 262K | $0.15 | $1.50 | Function calling |
| Vertex AI | MedGemma models | Varies | Varies | Varies | Medical-focused Gemma models on custom endpoints |
| Watson X | 27 new foundation models | Varies | Varies | Varies | Granite, Llama, Mistral families |
#### Features
- **[OpenAI](../../docs/providers/openai)**
- Add GPT-5 Pro model configuration and documentation - [PR #15258](https://github.com/BerriAI/litellm/pull/15258)
- Add stop parameter to non-supported params for GPT-5 - [PR #15244](https://github.com/BerriAI/litellm/pull/15244)
- Day 0 Support, Add gpt-image-1-mini - [PR #15259](https://github.com/BerriAI/litellm/pull/15259)
- Add gpt-realtime-mini support - [PR #15283](https://github.com/BerriAI/litellm/pull/15283)
- Add gpt-5-pro-2025-10-06 to model costs - [PR #15344](https://github.com/BerriAI/litellm/pull/15344)
- Minimal fix: gpt5 models should not go on cooldown when called with temperature!=1 - [PR #15330](https://github.com/BerriAI/litellm/pull/15330)
- **[Snowflake Cortex](../../docs/providers/snowflake)**
- Add function calling support for Snowflake Cortex REST API - [PR #15221](https://github.com/BerriAI/litellm/pull/15221)
- **[Gemini](../../docs/providers/gemini)**
- Fix header forwarding for Gemini/Vertex AI providers in proxy mode - [PR #15231](https://github.com/BerriAI/litellm/pull/15231)
- **[Azure](../../docs/providers/azure)**
- Removed stop param from unsupported azure models - [PR #15229](https://github.com/BerriAI/litellm/pull/15229)
- Fix(azure/responses): remove invalid status param from azure call - [PR #15253](https://github.com/BerriAI/litellm/pull/15253)
- Add new Azure AI models with pricing details - [PR #15387](https://github.com/BerriAI/litellm/pull/15387)
- AzureAD Default credentials - select credential type based on environment - [PR #14470](https://github.com/BerriAI/litellm/pull/14470)
- **[Bedrock](../../docs/providers/bedrock)**
- Add Global Cross-Region Inference - [PR #15210](https://github.com/BerriAI/litellm/pull/15210)
- Add Cohere Embed v4 support for AWS Bedrock - [PR #15298](https://github.com/BerriAI/litellm/pull/15298)
- Fix(bedrock): include cacheWriteInputTokens in prompt_tokens calculation - [PR #15292](https://github.com/BerriAI/litellm/pull/15292)
- Add Bedrock AU Cross-Region Inference for Claude Sonnet 4.5 - [PR #15402](https://github.com/BerriAI/litellm/pull/15402)
- Converse → /v1/messages streaming doesn't handle parallel tool calls with Claude models - [PR #15315](https://github.com/BerriAI/litellm/pull/15315)
- **[Vertex AI](../../docs/providers/vertex)**
- Implement Context Caching for Vertex AI provider - [PR #15226](https://github.com/BerriAI/litellm/pull/15226)
- Support for Vertex AI Gemma Models on Custom Endpoints - [PR #15397](https://github.com/BerriAI/litellm/pull/15397)
- VertexAI - gemma model family support (custom endpoints) - [PR #15419](https://github.com/BerriAI/litellm/pull/15419)
- VertexAI Gemma model family streaming support + Added MedGemma - [PR #15427](https://github.com/BerriAI/litellm/pull/15427)
- **[OCI](../../docs/providers/oci)**
- Add OCI Cohere support with tool calling and streaming capabilities - [PR #15365](https://github.com/BerriAI/litellm/pull/15365)
- **[Watson X](../../docs/providers/watsonx)**
- Add Watson X foundation model definitions to model_prices_and_context_window.json - [PR #15219](https://github.com/BerriAI/litellm/pull/15219)
- Watsonx - Apply correct prompt templates for openai/gpt-oss model family - [PR #15341](https://github.com/BerriAI/litellm/pull/15341)
- **[OpenRouter](../../docs/providers/openrouter)**
- Fix - (openrouter): move cache_control to content blocks for claude/gemini - [PR #15345](https://github.com/BerriAI/litellm/pull/15345)
- Fix - OpenRouter cache_control to only apply to last content block - [PR #15395](https://github.com/BerriAI/litellm/pull/15395)
- **[Together AI](../../docs/providers/togetherai)**
- Add new together models - [PR #15383](https://github.com/BerriAI/litellm/pull/15383)
### Bug Fixes
- **General**
- Bug fix: gpt-5-chat-latest has incorrect max_input_tokens value - [PR #15116](https://github.com/BerriAI/litellm/pull/15116)
- Fix reasoning response ID - [PR #15265](https://github.com/BerriAI/litellm/pull/15265)
- Fix issue with parsing assistant messages - [PR #15320](https://github.com/BerriAI/litellm/pull/15320)
- Fix litellm_param based costing - [PR #15336](https://github.com/BerriAI/litellm/pull/15336)
- Fix lint errors - [PR #15406](https://github.com/BerriAI/litellm/pull/15406)
---
## LLM API Endpoints
#### Features
- **[Responses API](../../docs/response_api)**
- Added streaming support for response api streaming image generation - [PR #15269](https://github.com/BerriAI/litellm/pull/15269)
- Add native Responses API support for litellm_proxy provider - [PR #15347](https://github.com/BerriAI/litellm/pull/15347)
- Temporarily relax ResponsesAPIResponse parsing to support custom backends (e.g., vLLM) - [PR #15362](https://github.com/BerriAI/litellm/pull/15362)
- **[Files API](../../docs/files_api)**
- Feat(files): add @client decorator to file operations - [PR #15339](https://github.com/BerriAI/litellm/pull/15339)
- **[/generateContent](../../docs/providers/gemini)**
- Fix gemini cli by actually streaming the response - [PR #15264](https://github.com/BerriAI/litellm/pull/15264)
- **[Azure Passthrough](../../docs/pass_through/azure)**
- Azure - passthrough support with router models - [PR #15240](https://github.com/BerriAI/litellm/pull/15240)
#### Bugs
- **General**
- Fix x-litellm-cache-key header not being returned on cache hit - [PR #15348](https://github.com/BerriAI/litellm/pull/15348)
---
## Management Endpoints / UI
#### Features
- **Proxy CLI Auth**
- Proxy CLI - dont store existing key in the URL, store it in the state param - [PR #15290](https://github.com/BerriAI/litellm/pull/15290)
- **Models + Endpoints**
- Make PATCH `/model/{model_id}/update` handle `team_id` consistently with POST `/model/new` - [PR #15297](https://github.com/BerriAI/litellm/pull/15297)
- Feature: adds Infinity as a provider in the UI - [PR #15285](https://github.com/BerriAI/litellm/pull/15285)
- Fix: model + endpoints page crash when config file contains router_settings.model_group_alias - [PR #15308](https://github.com/BerriAI/litellm/pull/15308)
- Models & Endpoints Initial Refactor - [PR #15435](https://github.com/BerriAI/litellm/pull/15435)
- Litellm UI API Reference page updates - [PR #15438](https://github.com/BerriAI/litellm/pull/15438)
- **Teams**
- Teams page: new column "Your Role" on the teams table - [PR #15384](https://github.com/BerriAI/litellm/pull/15384)
- LiteLLM Dashboard Teams UI refactor - [PR #15418](https://github.com/BerriAI/litellm/pull/15418)
- **UI Infrastructure**
- Added prettier to autoformat frontend - [PR #15215](https://github.com/BerriAI/litellm/pull/15215)
- Adds turbopack to the npm run dev command in UI to build faster during development - [PR #15250](https://github.com/BerriAI/litellm/pull/15250)
- (perf) fix: Replaces bloated key list calls with lean key aliases endpoint - [PR #15252](https://github.com/BerriAI/litellm/pull/15252)
- Potentially fixes a UI spasm issue with an expired cookie - [PR #15309](https://github.com/BerriAI/litellm/pull/15309)
- LiteLLM UI Refactor Infrastructure - [PR #15236](https://github.com/BerriAI/litellm/pull/15236)
- Enforces removal of unused imports from UI - [PR #15416](https://github.com/BerriAI/litellm/pull/15416)
- Fix: usage page >> Model Activity >> spend per day graph: y-axis clipping on large spend values - [PR #15389](https://github.com/BerriAI/litellm/pull/15389)
- Updates guardrail provider logos - [PR #15421](https://github.com/BerriAI/litellm/pull/15421)
- **Admin Settings**
- Fix: Router settings do not update despite success message - [PR #15249](https://github.com/BerriAI/litellm/pull/15249)
- Fix: Prevents DB from accidentally overriding config file values if they are empty in DB - [PR #15340](https://github.com/BerriAI/litellm/pull/15340)
- **SSO**
- SSO - support EntraID app roles - [PR #15351](https://github.com/BerriAI/litellm/pull/15351)
---
## Logging / Guardrail / Prompt Management Integrations
#### Features
- **[PostHog](../../docs/observability/posthog)**
- Feat: posthog per request api key - [PR #15379](https://github.com/BerriAI/litellm/pull/15379)
#### Guardrails
- **[EnkryptAI](../../docs/proxy/guardrails)**
- Add EnkryptAI Guardrails on LiteLLM - [PR #15390](https://github.com/BerriAI/litellm/pull/15390)
---
## Spend Tracking, Budgets and Rate Limiting
- **Tag Management**
- Tag Management - Add support for setting tag based budgets - [PR #15433](https://github.com/BerriAI/litellm/pull/15433)
- **Dynamic Rate Limiter v3**
- QA/Fixes - Dynamic Rate Limiter v3 - final QA - [PR #15311](https://github.com/BerriAI/litellm/pull/15311)
- Fix dynamic Rate limiter v3 - inserting litellm_model_saturation - [PR #15394](https://github.com/BerriAI/litellm/pull/15394)
- **Shared Health Check**
- Implement Shared Health Check State Across Pods - [PR #15380](https://github.com/BerriAI/litellm/pull/15380)
---
## MCP Gateway
- **Tool Control**
- MCP Gateway - UI - Select allowed tools for Key, Teams - [PR #15241](https://github.com/BerriAI/litellm/pull/15241)
- MCP Gateway - Backend - Allow storing allowed tools by team/key - [PR #15243](https://github.com/BerriAI/litellm/pull/15243)
- MCP Gateway - Fine-grained Database Object Storage Control - [PR #15255](https://github.com/BerriAI/litellm/pull/15255)
- MCP Gateway - Litellm mcp fixes team control - [PR #15304](https://github.com/BerriAI/litellm/pull/15304)
- MCP Gateway - QA/Fixes - Ensure Team/Key level enforcement works for MCPs - [PR #15305](https://github.com/BerriAI/litellm/pull/15305)
- Feature: Include server_name in /v1/mcp/server/health endpoint response - [PR #15431](https://github.com/BerriAI/litellm/pull/15431)
- **OpenAPI Integration**
- MCP - support converting OpenAPI specs to MCP servers - [PR #15343](https://github.com/BerriAI/litellm/pull/15343)
- MCP - specify allowed params per tool - [PR #15346](https://github.com/BerriAI/litellm/pull/15346)
- **Configuration**
- MCP - support setting CA_BUNDLE_PATH - [PR #15253](https://github.com/BerriAI/litellm/pull/15253)
- Fix: Ensure MCP client stays open during tool call - [PR #15391](https://github.com/BerriAI/litellm/pull/15391)
- Remove hardcoded "public" schema in migration.sql - [PR #15363](https://github.com/BerriAI/litellm/pull/15363)
---
## Performance / Loadbalancing / Reliability improvements
- **Router Optimizations**
- Fix - Router: add model_name index for O(1) deployment lookups - [PR #15113](https://github.com/BerriAI/litellm/pull/15113)
- Refactor Utils: extract inner function from client - [PR #15234](https://github.com/BerriAI/litellm/pull/15234)
- Fix Networking: remove limitations - [PR #15302](https://github.com/BerriAI/litellm/pull/15302)
- **Session Management**
- Fix - Sessions not being shared - [PR #15388](https://github.com/BerriAI/litellm/pull/15388)
- Fix: remove panic from hot path - [PR #15396](https://github.com/BerriAI/litellm/pull/15396)
- Fix - shared session parsing and usage issue - [PR #15440](https://github.com/BerriAI/litellm/pull/15440)
- Fix: handle closed aiohttp sessions - [PR #15442](https://github.com/BerriAI/litellm/pull/15442)
- Fix: prevent session leaks when recreating aiohttp sessions - [PR #15443](https://github.com/BerriAI/litellm/pull/15443)
- **SSL/TLS Performance**
- Perf: optimize SSL/TLS handshake performance with prioritized cipher - [PR #15398](https://github.com/BerriAI/litellm/pull/15398)
- **Dependencies**
- Upgrades tenacity version to 8.5.0 - [PR #15303](https://github.com/BerriAI/litellm/pull/15303)
- **Data Masking**
- Fix - SensitiveDataMasker converts lists to string - [PR #15420](https://github.com/BerriAI/litellm/pull/15420)
---
## General AI Gateway Improvements
#### Security
- **General**
- Fix: redact AWS credentials when redact_user_api_key_info enabled - [PR #15321](https://github.com/BerriAI/litellm/pull/15321)
---
## Documentation Updates
- **Provider Documentation**
- Update doc: perf update - [PR #15211](https://github.com/BerriAI/litellm/pull/15211)
- Add W&B Inference documentation - [PR #15278](https://github.com/BerriAI/litellm/pull/15278)
- **Deployment**
- Deletion of docker-compose buggy comment that cause `config.yaml` based startup fail - [PR #15425](https://github.com/BerriAI/litellm/pull/15425)
---
## New Contributors
* @Gal-bloch made their first contribution in [PR #15219](https://github.com/BerriAI/litellm/pull/15219)
* @lcfyi made their first contribution in [PR #15315](https://github.com/BerriAI/litellm/pull/15315)
* @ashengstd made their first contribution in [PR #15362](https://github.com/BerriAI/litellm/pull/15362)
* @vkolehmainen made their first contribution in [PR #15363](https://github.com/BerriAI/litellm/pull/15363)
* @jlan-nl made their first contribution in [PR #15330](https://github.com/BerriAI/litellm/pull/15330)
* @BCook98 made their first contribution in [PR #15402](https://github.com/BerriAI/litellm/pull/15402)
* @PabloGmz96 made their first contribution in [PR #15425](https://github.com/BerriAI/litellm/pull/15425)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.77.7.rc.1...v1.78.0.rc.1)**

50
docs/my-website/sidebars.js
View file

@ -36,6 +36,7 @@ const sidebars = {
"proxy/guardrails/aporia_api",
"proxy/guardrails/azure_content_guardrail",
"proxy/guardrails/bedrock",
"proxy/guardrails/enkryptai",
"proxy/guardrails/lasso_security",
"proxy/guardrails/guardrails_ai",
"proxy/guardrails/lakera_ai",
@ -93,10 +94,10 @@ const sidebars = {
{
type: "category",
label: "LiteLLM Proxy Server",
label: "LiteLLM AI Gateway",
link: {
type: "generated-index",
title: "LiteLLM Proxy Server (LLM Gateway)",
title: "LiteLLM AI Gateway (LLM Proxy)",
description: `OpenAI Proxy Server (LLM Gateway) to call 100+ LLMs in a unified interface & track spend, set budgets per virtual key/user`,
slug: "/simple_proxy",
},
@ -188,12 +189,13 @@ const sidebars = {
type: "category",
label: "Budgets + Rate Limits",
items: [
"proxy/users",
"proxy/team_budgets",
"proxy/tag_budgets",
"proxy/customers",
"proxy/dynamic_rate_limit",
"proxy/rate_limit_tiers",
"proxy/team_budgets",
"proxy/temporary_budget_increase",
"proxy/users"
],
},
"proxy/caching",
@ -333,8 +335,19 @@ const sidebars = {
"image_variations",
]
},
"mcp",
{
type: "category",
label: "/mcp - Model Context Protocol",
items: [
"mcp",
"mcp_usage",
"mcp_control",
"mcp_cost",
"mcp_guardrail",
]
},
"moderation",
"ocr",
{
type: "category",
label: "Pass-through Endpoints (Anthropic SDK, etc.)",
@ -412,6 +425,7 @@ const sidebars = {
items: [
"providers/vertex",
"providers/vertex_partner",
"providers/vertex_self_deployed",
"providers/vertex_image",
"providers/vertex_batch",
]
@ -458,7 +472,14 @@ const sidebars = {
"providers/deepgram",
"providers/watsonx",
"providers/predibase",
"providers/nvidia_nim",
{
type: "category",
label: "Nvidia NIM",
items: [
"providers/nvidia_nim",
"providers/nvidia_nim_rerank",
]
},
{ type: "doc", id: "providers/nscale", label: "Nscale (EU Sovereign)" },
"providers/xai",
"providers/moonshot",
@ -515,22 +536,18 @@ const sidebars = {
"providers/oci",
"providers/datarobot",
"providers/ovhcloud",
"providers/wandb_inference",
"providers/cometapi",
],
},
{
type: "category",
label: "Guides",
items: [
{
type: "category",
label: "Tools",
items: [
"completion/computer_use",
"completion/web_search",
"completion/web_fetch",
"completion/function_call",
]
},
"completion/computer_use",
"completion/web_search",
"completion/web_fetch",
"completion/function_call",
"completion/audio",
"completion/document_understanding",
"completion/drop_params",
@ -667,6 +684,7 @@ const sidebars = {
items: [
"data_security",
"data_retention",
"proxy/security_encryption_faq",
"migration_policy",
{
type: "category",

1
docs/my-website/src/pages/completion/supported.md
View file

@ -8,6 +8,7 @@
| gpt-3.5-turbo-16k | `completion('gpt-3.5-turbo-16k', messages)` | `os.environ['OPENAI_API_KEY']` |
| gpt-3.5-turbo-16k-0613 | `completion('gpt-3.5-turbo-16k-0613', messages)` | `os.environ['OPENAI_API_KEY']` |
| gpt-4 | `completion('gpt-4', messages)` | `os.environ['OPENAI_API_KEY']` |
| gpt-5-pro | `completion('gpt-5-pro', messages)` | `os.environ['OPENAI_API_KEY']` |
## Azure OpenAI Chat Completion Models
For Azure calls add the `azure/` prefix to `model`. If your azure deployment name is `gpt-v-2` set `model` = `azure/gpt-v-2`

8
enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py
View file

@ -119,6 +119,7 @@ class PagerDutyAlerting(SlackAlerting):
user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"),
user_api_key_user_email=_meta.get("user_api_key_user_email"),
user_api_key_request_route=_meta.get("user_api_key_request_route"),
user_api_key_auth_metadata=_meta.get("user_api_key_auth_metadata"),
)
)
@ -196,7 +197,11 @@ class PagerDutyAlerting(SlackAlerting):
user_api_key_alias=user_api_key_dict.key_alias,
user_api_key_spend=user_api_key_dict.spend,
user_api_key_max_budget=user_api_key_dict.max_budget,
user_api_key_budget_reset_at=user_api_key_dict.budget_reset_at.isoformat() if user_api_key_dict.budget_reset_at else None,
user_api_key_budget_reset_at=(
user_api_key_dict.budget_reset_at.isoformat()
if user_api_key_dict.budget_reset_at
else None
),
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_team_id=user_api_key_dict.team_id,
user_api_key_user_id=user_api_key_dict.user_id,
@ -204,6 +209,7 @@ class PagerDutyAlerting(SlackAlerting):
user_api_key_end_user_id=user_api_key_dict.end_user_id,
user_api_key_user_email=user_api_key_dict.user_email,
user_api_key_request_route=user_api_key_dict.request_route,
user_api_key_auth_metadata=user_api_key_dict.metadata,
)
)

38
enterprise/litellm_enterprise/integrations/prometheus.py
View file

@ -21,6 +21,7 @@ from litellm._logging import print_verbose, verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import LiteLLM_TeamTable, UserAPIKeyAuth
from litellm.types.integrations.prometheus import *
from litellm.types.integrations.prometheus import _sanitize_prometheus_label_name
from litellm.types.utils import StandardLoggingPayload
from litellm.utils import get_end_user_id_for_cost_tracking
@ -794,9 +795,16 @@ class PrometheusLogger(CustomLogger):
output_tokens = standard_logging_payload["completion_tokens"]
tokens_used = standard_logging_payload["total_tokens"]
response_cost = standard_logging_payload["response_cost"]
_requester_metadata = standard_logging_payload["metadata"].get(
_requester_metadata: Optional[dict] = standard_logging_payload["metadata"].get(
"requester_metadata"
)
user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[
"metadata"
].get("user_api_key_auth_metadata")
combined_metadata: Dict[str, Any] = {
**(_requester_metadata if _requester_metadata else {}),
**(user_api_key_auth_metadata if user_api_key_auth_metadata else {}),
}
if standard_logging_payload is not None and isinstance(
standard_logging_payload, dict
):
@ -828,8 +836,7 @@ class PrometheusLogger(CustomLogger):
exception_status=None,
exception_class=None,
custom_metadata_labels=get_custom_labels_from_metadata(
metadata=standard_logging_payload["metadata"].get("requester_metadata")
or {}
metadata=combined_metadata
),
route=standard_logging_payload["metadata"].get(
"user_api_key_request_route"
@ -1649,9 +1656,22 @@ class PrometheusLogger(CustomLogger):
api_base: Optional[str],
api_provider: str,
):
self.litellm_deployment_state.labels(
litellm_model_name, model_id, api_base, api_provider
).set(state)
"""
Set the deployment state.
"""
### get labels
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_deployment_state"
),
enum_values=UserAPIKeyLabelValues(
litellm_model_name=litellm_model_name,
model_id=model_id,
api_base=api_base,
api_provider=api_provider,
),
)
self.litellm_deployment_state.labels(**_labels).set(state)
def set_deployment_healthy(
self,
@ -2228,8 +2248,10 @@ def prometheus_label_factory(
if enum_values.custom_metadata_labels is not None:
for key, value in enum_values.custom_metadata_labels.items():
if key in supported_enum_labels:
filtered_labels[key] = value
# check sanitized key
sanitized_key = _sanitize_prometheus_label_name(key)
if sanitized_key in supported_enum_labels:
filtered_labels[sanitized_key] = value
# Add custom tags if configured
if enum_values.tags is not None:

4
enterprise/litellm_enterprise/proxy/guardrails/endpoints.py
View file

@ -36,6 +36,8 @@ async def apply_guardrail(
if active_guardrail is None:
raise Exception(f"Guardrail {request.guardrail_name} not found")
return await active_guardrail.apply_guardrail(
response_text = await active_guardrail.apply_guardrail(
text=request.text, language=request.language, entities=request.entities
)
return ApplyGuardrailResponse(response_text=response_text)

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.23-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.23.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.25-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.25.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.26-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.26.tar.gz vendored Normal file
View file

Binary file not shown.

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20250918083359_drop_spec_version_column_from_mcp_table/migration.sql
View file

@ -5,4 +5,4 @@
*/
-- AlterTable
ALTER TABLE "public"."LiteLLM_MCPServerTable" DROP COLUMN "spec_version";
ALTER TABLE "LiteLLM_MCPServerTable" DROP COLUMN "spec_version";

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20251003165142_add_allowed_tools_to_mcp/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "allowed_tools" TEXT[] DEFAULT ARRAY[]::TEXT[];

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20251003190954_extra_headers_to_mcp_table/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "extra_headers" TEXT[] DEFAULT ARRAY[]::TEXT[];

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20251006143948_add_mcp_tool_permissions/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN "mcp_tool_permissions" JSONB;

18
litellm-proxy-extras/litellm_proxy_extras/migrations/20251011084309_add_tag_table/migration.sql Normal file
View file

@ -0,0 +1,18 @@
-- CreateTable
CREATE TABLE "LiteLLM_TagTable" (
"tag_name" TEXT NOT NULL,
"description" TEXT,
"models" TEXT[],
"model_info" JSONB,
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
"budget_id" TEXT,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "LiteLLM_TagTable_pkey" PRIMARY KEY ("tag_name")
);
-- AddForeignKey
ALTER TABLE "LiteLLM_TagTable" ADD CONSTRAINT "LiteLLM_TagTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;

18
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -25,6 +25,7 @@ model LiteLLM_BudgetTable {
organization LiteLLM_OrganizationTable[] // multiple orgs can have the same budget
keys LiteLLM_VerificationToken[] // multiple keys can have the same budget
end_users LiteLLM_EndUserTable[] // multiple end-users can have the same budget
tags LiteLLM_TagTable[] // multiple tags can have the same budget
team_membership LiteLLM_TeamMembership[] // budgets of Users within a Team
organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization
}
@ -156,6 +157,7 @@ model LiteLLM_ObjectPermissionTable {
object_permission_id String @id @default(uuid())
mcp_servers String[] @default([])
mcp_access_groups String[] @default([])
mcp_tool_permissions Json? // Tool-level permissions for MCP servers. Format: {"server_id": ["tool_name_1", "tool_name_2"]}
vector_stores String[] @default([])
teams LiteLLM_TeamTable[]
verification_tokens LiteLLM_VerificationToken[]
@ -178,6 +180,8 @@ model LiteLLM_MCPServerTable {
updated_by String?
mcp_info Json? @default("{}")
mcp_access_groups String[]
allowed_tools String[] @default([])
extra_headers String[] @default([])
// Health check status
status String? @default("unknown")
last_health_check DateTime?
@ -242,6 +246,20 @@ model LiteLLM_EndUserTable {
blocked Boolean @default(false)
}
// Track tags with budgets and spend
model LiteLLM_TagTable {
tag_name String @id
description String?
models String[]
model_info Json? // maps model_id to model_name
spend Float @default(0.0)
budget_id String?
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
created_at DateTime @default(now()) @map("created_at")
created_by String?
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
}
// store proxy config.yaml
model LiteLLM_Config {
param_name String @id

13
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -131,7 +131,9 @@ class ProxyExtrasDBManager:
)
@staticmethod
def _resolve_all_migrations(migrations_dir: str, schema_path: str):
def _resolve_all_migrations(
migrations_dir: str, schema_path: str, mark_all_applied: bool = True
):
"""
1. Compare the current database state to schema.prisma and generate a migration for the diff.
2. Run prisma migrate deploy to apply any pending migrations.
@ -210,6 +212,8 @@ class ProxyExtrasDBManager:
logger.warning("Migration diff application timed out.")
# 3. Mark all migrations as applied
if not mark_all_applied:
return
migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir)
logger.info(f"Resolving {len(migration_names)} migrations")
for migration_name in migration_names:
@ -263,6 +267,13 @@ class ProxyExtrasDBManager:
logger.info(f"prisma migrate deploy stdout: {result.stdout}")
logger.info("prisma migrate deploy completed")
# Run sanity check to ensure DB matches schema
logger.info("Running post-migration sanity check...")
ProxyExtrasDBManager._resolve_all_migrations(
migrations_dir, schema_path, mark_all_applied=False
)
logger.info("✅ Post-migration sanity check completed")
return True
except subprocess.CalledProcessError as e:
logger.info(f"prisma db error: {e.stderr}, e: {e.stdout}")

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.2.22"
version = "0.2.27"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.2.22"
version = "0.2.27"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

16
litellm/__init__.py
View file

@ -290,7 +290,7 @@ banned_keywords_list: Optional[Union[str, List]] = None
llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all"
guardrail_name_config_map: Dict[str, GuardrailItem] = {}
include_cost_in_streaming_usage: bool = False
### PROMPTS ###
### PROMPTS ####
from litellm.types.prompts.init_prompts import PromptSpec
prompt_name_config_map: Dict[str, PromptSpec] = {}
@ -367,7 +367,7 @@ disable_add_prefix_to_prompt: bool = (
disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
public_model_groups: Optional[List[str]] = None
public_model_groups_links: Dict[str, str] = {}
#### REQUEST PRIORITIZATION ######
#### REQUEST PRIORITIZATION #######
priority_reservation: Optional[Dict[str, float]] = None
priority_reservation_settings: "PriorityReservationSettings" = (
PriorityReservationSettings()
@ -497,6 +497,7 @@ azure_text_models: Set = set()
anyscale_models: Set = set()
cerebras_models: Set = set()
galadriel_models: Set = set()
nvidia_nim_models: Set = set()
sambanova_models: Set = set()
sambanova_embedding_models: Set = set()
novita_models: Set = set()
@ -691,6 +692,8 @@ def add_known_models():
cerebras_models.add(key)
elif value.get("litellm_provider") == "galadriel":
galadriel_models.add(key)
elif value.get("litellm_provider") == "nvidia_nim":
nvidia_nim_models.add(key)
elif value.get("litellm_provider") == "sambanova":
sambanova_models.add(key)
elif value.get("litellm_provider") == "sambanova-embedding-models":
@ -818,6 +821,7 @@ model_list = list(
| anyscale_models
| cerebras_models
| galadriel_models
| nvidia_nim_models
| sambanova_models
| azure_text_models
| novita_models
@ -901,6 +905,7 @@ models_by_provider: dict = {
"anyscale": anyscale_models,
"cerebras": cerebras_models,
"galadriel": galadriel_models,
"nvidia_nim": nvidia_nim_models,
"sambanova": sambanova_models | sambanova_embedding_models,
"novita": novita_models,
"nebius": nebius_models | nebius_embedding_models,
@ -1061,6 +1066,7 @@ from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig
from .llms.infinity.rerank.transformation import InfinityRerankConfig
from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig
from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig
from .llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig
from .llms.clarifai.chat.transformation import ClarifaiConfig
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
from .llms.meta_llama.chat.transformation import LlamaAPIConfig
@ -1161,6 +1167,7 @@ from .llms.bedrock.embed.amazon_titan_v2_transformation import (
)
from .llms.cohere.chat.transformation import CohereChatConfig
from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConfig
from .llms.bedrock.embed.twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig
from .llms.openai.openai import OpenAIConfig, MistralEmbeddingConfig
from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig
from .llms.deepinfra.chat.transformation import DeepInfraConfig
@ -1183,6 +1190,9 @@ from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
from .llms.azure.responses.o_series_transformation import (
AzureOpenAIOSeriesResponsesAPIConfig,
)
from .llms.litellm_proxy.responses.transformation import (
LiteLLMProxyResponsesAPIConfig,
)
from .llms.openai.chat.o_series_transformation import (
OpenAIOSeriesConfig as OpenAIO1Config, # maintain backwards compatibility
OpenAIOSeriesConfig,
@ -1277,6 +1287,7 @@ from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig
from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig
from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig
from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig
from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig
from .llms.lemonade.chat.transformation import LemonadeChatConfig
from .main import * # type: ignore
from .integrations import *
@ -1314,6 +1325,7 @@ from .batch_completion.main import * # type: ignore
from .rerank_api.main import *
from .llms.anthropic.experimental_pass_through.messages.handler import *
from .responses.main import *
from .ocr.main import *
from .realtime_api.main import _arealtime
from .fine_tuning.main import *
from .files.main import *

19
litellm/_redis.py
View file

@ -177,14 +177,21 @@ def get_redis_url_from_environment():
raise ValueError(
"Either 'REDIS_URL' or both 'REDIS_HOST' and 'REDIS_PORT' must be specified for Redis."
)
if "REDIS_PASSWORD" in os.environ:
redis_password = f":{os.environ['REDIS_PASSWORD']}@"
if "REDIS_SSL" in os.environ and os.environ["REDIS_SSL"].lower() == "true":
redis_protocol = "rediss"
else:
redis_password = ""
redis_protocol = "redis"
# Build authentication part of URL
auth_part = ""
if "REDIS_USERNAME" in os.environ and "REDIS_PASSWORD" in os.environ:
auth_part = f"{os.environ['REDIS_USERNAME']}:{os.environ['REDIS_PASSWORD']}@"
elif "REDIS_PASSWORD" in os.environ:
auth_part = f"{os.environ['REDIS_PASSWORD']}@"
return (
f"redis://{redis_password}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
)

144
litellm/batches/main.py
View file

@ -59,18 +59,22 @@ def _resolve_timeout(
) -> float:
"""
Resolve timeout value from various sources and handle httpx.Timeout objects.
Args:
optional_params: GenericLiteLLMParams object containing timeout
kwargs: Additional kwargs that may contain request_timeout
custom_llm_provider: Provider name for httpx timeout support check
default_timeout: Default timeout value to use
Returns:
Resolved timeout as float
"""
timeout = optional_params.timeout or kwargs.get("request_timeout", default_timeout) or default_timeout
timeout = (
optional_params.timeout
or kwargs.get("request_timeout", default_timeout)
or default_timeout
)
# Handle httpx.Timeout objects
if isinstance(timeout, httpx.Timeout):
if supports_httpx_timeout(custom_llm_provider) is False:
@ -81,11 +85,11 @@ def _resolve_timeout(
# For providers that support httpx.Timeout, we still need to return a float
# This case might need to be handled differently based on the actual use case
return float(timeout.read or default_timeout)
# Handle None case
if timeout is None:
return float(default_timeout)
# Handle numeric values (int, float, string representations)
return float(timeout)
@ -163,15 +167,19 @@ def create_batch(
try:
if model is not None:
model, _, _, _ = get_llm_provider(
model=model,
custom_llm_provider=None,
)
model=model,
custom_llm_provider=None,
)
except Exception as e:
verbose_logger.exception(f"litellm.batches.main.py::create_batch() - Error inferring custom_llm_provider - {str(e)}")
verbose_logger.exception(
f"litellm.batches.main.py::create_batch() - Error inferring custom_llm_provider - {str(e)}"
)
_is_async = kwargs.pop("acreate_batch", False) is True
litellm_params = dict(GenericLiteLLMParams(**kwargs))
litellm_logging_obj: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, kwargs.get("litellm_logging_obj", None))
litellm_logging_obj: LiteLLMLoggingObj = cast(
LiteLLMLoggingObj, kwargs.get("litellm_logging_obj", None)
)
### TIMEOUT LOGIC ###
timeout = _resolve_timeout(optional_params, kwargs, custom_llm_provider)
litellm_logging_obj.update_environment_variables(
@ -189,7 +197,6 @@ def create_batch(
},
custom_llm_provider=custom_llm_provider,
)
_create_batch_request = CreateBatchRequest(
completion_window=completion_window,
@ -378,6 +385,7 @@ async def aretrieve_batch(
except Exception as e:
raise e
def _handle_retrieve_batch_providers_without_provider_config(
batch_id: str,
optional_params: GenericLiteLLMParams,
@ -497,6 +505,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
)
return response
@client
def retrieve_batch(
batch_id: str,
@ -513,7 +522,9 @@ def retrieve_batch(
"""
try:
optional_params = GenericLiteLLMParams(**kwargs)
litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj", None)
litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get(
"litellm_logging_obj", None
)
### TIMEOUT LOGIC ###
timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
litellm_params = get_litellm_params(
@ -549,7 +560,26 @@ def retrieve_batch(
_is_async = kwargs.pop("aretrieve_batch", False) is True
client = kwargs.get("client", None)
# Check if this is an async invoke ARN (different from regular batch ARN)
# Async invoke ARNs have format: arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:[0-9]{12}:async-invoke/[a-z0-9]{12}
if (
batch_id.startswith("arn:aws")
and ":bedrock:" in batch_id
and ":async-invoke/" in batch_id
):
# Handle async invoke status check
# Remove aws_region_name from kwargs to avoid duplicate parameter
async_kwargs = kwargs.copy()
async_kwargs.pop("aws_region_name", None)
return _handle_async_invoke_status(
batch_id=batch_id,
aws_region_name=kwargs.get("aws_region_name", "us-east-1"),
logging_obj=litellm_logging_obj,
**async_kwargs,
)
# Try to use provider config first (for providers like bedrock)
model: Optional[str] = kwargs.get("model", None)
if model is not None:
@ -559,7 +589,7 @@ def retrieve_batch(
)
else:
provider_config = None
if provider_config is not None:
response = base_llm_http_handler.retrieve_batch(
batch_id=batch_id,
@ -568,7 +598,8 @@ def retrieve_batch(
headers=extra_headers or {},
api_base=optional_params.api_base,
api_key=optional_params.api_key,
logging_obj=litellm_logging_obj or LiteLLMLoggingObj(
logging_obj=litellm_logging_obj
or LiteLLMLoggingObj(
model=model or "bedrock/unknown",
messages=[],
stream=False,
@ -586,7 +617,6 @@ def retrieve_batch(
model=model,
)
return response
#########################################################
# Handle providers without provider config
@ -600,7 +630,7 @@ def retrieve_batch(
_is_async=_is_async,
timeout=timeout,
)
except Exception as e:
raise e
@ -933,3 +963,79 @@ def cancel_batch(
return response
except Exception as e:
raise e
def _handle_async_invoke_status(
batch_id: str, aws_region_name: str, logging_obj=None, **kwargs
) -> "LiteLLMBatch":
"""
Handle async invoke status check for AWS Bedrock.
Args:
batch_id: The async invoke ARN
aws_region_name: AWS region name
**kwargs: Additional parameters
Returns:
dict: Status information including status, output_file_id (S3 URL), etc.
"""
import asyncio
from litellm.llms.bedrock.embed.embedding import BedrockEmbedding
async def _async_get_status():
# Create embedding handler instance
embedding_handler = BedrockEmbedding()
# Get the status of the async invoke job
status_response = await embedding_handler._get_async_invoke_status(
invocation_arn=batch_id,
aws_region_name=aws_region_name,
logging_obj=logging_obj,
**kwargs,
)
# Transform response to a LiteLLMBatch object
from litellm.types.utils import LiteLLMBatch
result = LiteLLMBatch(
id=status_response["invocationArn"],
object="batch",
status=status_response["status"],
created_at=status_response["submitTime"],
in_progress_at=status_response["lastModifiedTime"],
completed_at=status_response.get("endTime"),
failed_at=status_response.get("endTime")
if status_response["status"] == "failed"
else None,
request_counts={
"total": 1,
"completed": 1 if status_response["status"] == "completed" else 0,
"failed": 1 if status_response["status"] == "failed" else 0,
},
metadata={
"output_file_id": status_response["outputDataConfig"][
"s3OutputDataConfig"
]["s3Uri"],
"failure_message": status_response.get("failureMessage"),
"model_arn": status_response["modelArn"],
},
)
return result
# Since this function is called from within an async context via run_in_executor,
# we need to create a new event loop in a thread to avoid conflicts
import concurrent.futures
def run_in_thread():
new_loop = asyncio.new_event_loop()
asyncio.set_event_loop(new_loop)
try:
return new_loop.run_until_complete(_async_get_status())
finally:
new_loop.close()
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(run_in_thread)
return future.result()

91
litellm/caching/caching_handler.py
View file

@ -17,6 +17,7 @@ In each method it will call the appropriate method from caching.py
import asyncio
import datetime
import inspect
import time
from typing import (
TYPE_CHECKING,
Any,
@ -57,10 +58,14 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.utils import CustomStreamWrapper
else:
LiteLLMLoggingObj = Any
CustomStreamWrapper = Any
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
)
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
class CachingHandlerResponse(BaseModel):
@ -112,7 +117,7 @@ class LLMCachingHandler:
call_type: str,
kwargs: Dict[str, Any],
args: Optional[Tuple[Any, ...]] = None,
) -> CachingHandlerResponse:
) -> Optional[CachingHandlerResponse]:
"""
Internal method to get from the cache.
Handles different call types (embeddings, chat/completions, text_completion, transcription)
@ -133,32 +138,27 @@ class LLMCachingHandler:
Raises:
None
"""
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
)
from litellm.utils import CustomStreamWrapper
kwargs = kwargs.copy()
args = args or ()
#########################################################
# Init cache timing metrics
#########################################################
cache_check_start_time = datetime.datetime.now()
cache_check_end_time = None
#########################################################
parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs)
kwargs["parent_otel_span"] = parent_otel_span
final_embedding_cached_response: Optional[EmbeddingResponse] = None
embedding_all_elements_cache_hit: bool = False
cached_result: Optional[Any] = None
# Check if caching should be performed BEFORE doing expensive operations
if (
(kwargs.get("caching", None) is None and litellm.cache is not None)
or kwargs.get("caching", False) is True
) and (
kwargs.get("cache", {}).get("no-cache", False) is not True
): # allow users to control returning cached responses from the completion function
args = args or ()
final_embedding_cached_response: Optional[EmbeddingResponse] = None
embedding_all_elements_cache_hit: bool = False
cached_result: Optional[Any] = None
kwargs = kwargs.copy()
#########################################################
# Init cache timing metrics
#########################################################
cache_check_start_time = time.perf_counter()
cache_check_end_time: Optional[float] = None
#########################################################
parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs)
kwargs["parent_otel_span"] = parent_otel_span
if litellm.cache is not None and self._is_call_type_supported_by_cache(
original_function=original_function
):
@ -168,7 +168,7 @@ class LLMCachingHandler:
kwargs=kwargs,
args=args,
)
cache_check_end_time = datetime.datetime.now()
cache_check_end_time = time.perf_counter()
if cached_result is not None and not isinstance(cached_result, list):
verbose_logger.debug("Cache Hit!")
@ -180,7 +180,7 @@ class LLMCachingHandler:
api_base=kwargs.get("api_base", None),
api_key=kwargs.get("api_key", None),
)
cache_duration_ms = (cache_check_end_time - cache_check_start_time).total_seconds() * 1000
cache_duration_ms = (cache_check_end_time - cache_check_start_time) * 1000
self._update_litellm_logging_obj_environment(
logging_obj=logging_obj,
model=model,
@ -212,9 +212,7 @@ class LLMCachingHandler:
end_time=end_time,
cache_hit=cache_hit,
)
cache_key = litellm.cache._get_preset_cache_key_from_kwargs(
**kwargs
)
cache_key = litellm.cache.get_cache_key(**kwargs)
if (
isinstance(cached_result, BaseModel)
or isinstance(cached_result, CustomStreamWrapper)
@ -245,11 +243,14 @@ class LLMCachingHandler:
final_embedding_cached_response=final_embedding_cached_response,
embedding_all_elements_cache_hit=embedding_all_elements_cache_hit,
)
verbose_logger.debug(f"CACHE RESULT: {cached_result}")
return CachingHandlerResponse(
cached_result=cached_result,
final_embedding_cached_response=final_embedding_cached_response,
)
verbose_logger.debug(f"CACHE RESULT: {cached_result}")
return CachingHandlerResponse(
cached_result=cached_result,
final_embedding_cached_response=final_embedding_cached_response,
)
# Caching disabled - return None to indicate no caching attempted
return None
def _sync_get_cache(
self,
@ -263,18 +264,22 @@ class LLMCachingHandler:
) -> CachingHandlerResponse:
from litellm.utils import CustomStreamWrapper
args = args or ()
new_kwargs = kwargs.copy()
new_kwargs.update(
convert_args_to_kwargs(
self.original_function,
args,
)
)
cached_result: Optional[Any] = None
# Check if caching should be performed BEFORE doing expensive kwargs copy
if litellm.cache is not None and self._is_call_type_supported_by_cache(
original_function=original_function
):
args = args or ()
# Now that we confirmed caching will happen, prepare kwargs
new_kwargs = kwargs.copy()
new_kwargs.update(
convert_args_to_kwargs(
self.original_function,
args,
)
)
print_verbose("Checking Sync Cache")
cached_result = litellm.cache.get_cache(**new_kwargs)
if cached_result is not None:
@ -321,9 +326,7 @@ class LLMCachingHandler:
end_time=end_time,
cache_hit=cache_hit
)
cache_key = litellm.cache._get_preset_cache_key_from_kwargs(
**kwargs
)
cache_key = litellm.cache.get_cache_key(**kwargs)
if (
isinstance(cached_result, BaseModel)
or isinstance(cached_result, CustomStreamWrapper)

100
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -18,13 +18,15 @@ from typing import (
cast,
)
from openai.types.responses.tool_param import FunctionToolParam
from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
CompletionTransformationBridge,
)
from litellm.types.llms.openai import Reasoning
from litellm.types.llms.openai import ChatCompletionToolParamFunctionChunk, Reasoning
if TYPE_CHECKING:
from openai.types.responses import ResponseInputImageParam
@ -50,6 +52,45 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def __init__(self):
pass
def _handle_raw_dict_response_item(
self, item: Dict[str, Any], index: int
) -> Tuple[Optional[Any], int]:
"""
Handle raw dict response items from Responses API (e.g., GPT-5 Codex format).
Args:
item: Raw dict response item with 'type' field
index: Current choice index
Returns:
Tuple of (Choice object or None, updated index)
"""
from litellm.types.utils import Choices, Message
item_type = item.get("type")
# Ignore reasoning items for now
if item_type == "reasoning":
return None, index
# Handle message items with output_text content
if item_type == "message":
content_list = item.get("content", [])
for content_item in content_list:
if isinstance(content_item, dict):
content_type = content_item.get("type")
if content_type == "output_text":
response_text = content_item.get("text", "")
msg = Message(
role=item.get("role", "assistant"),
content=response_text if response_text else "",
)
choice = Choices(message=msg, finish_reason="stop", index=index)
return choice, index + 1
# Unknown or unsupported type
return None, index
def convert_chat_completion_messages_to_responses_api(
self, messages: List["AllMessageValues"]
) -> Tuple[List[Any], Optional[str]]:
@ -201,6 +242,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if value is not None:
if key == "instructions" and instructions:
request_data["instructions"] = instructions
elif key == "stream_options" and isinstance(value, dict):
request_data["stream_options"] = value.get("include_obfuscation")
elif key == "user": # string can't be longer than 64 characters
if isinstance(value, str) and len(value) <= 64:
request_data["user"] = value
else:
request_data[key] = value
@ -221,7 +267,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
json_mode: Optional[bool] = None,
) -> "ModelResponse":
"""Transform Responses API response to chat completion response"""
from openai.types.responses import (
ResponseFunctionToolCall,
ResponseOutputMessage,
@ -240,19 +285,35 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
choices: List[Choices] = []
index = 0
reasoning_content: Optional[str] = None
for item in raw_response.output:
if isinstance(item, ResponseReasoningItem):
pass # ignore for now.
for summary_item in item.summary:
response_text = getattr(summary_item, "text", "")
reasoning_content = response_text if response_text else ""
elif isinstance(item, ResponseOutputMessage):
for content in item.content:
response_text = getattr(content, "text", "")
msg = Message(
role=item.role, content=response_text if response_text else ""
role=item.role,
content=response_text if response_text else "",
reasoning_content=reasoning_content,
)
choices.append(
Choices(message=msg, finish_reason="stop", index=index)
Choices(
message=msg,
finish_reason="stop",
index=index,
)
)
reasoning_content = None # flush reasoning content
index += 1
elif isinstance(item, ResponseFunctionToolCall):
msg = Message(
@ -267,12 +328,21 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
"type": "function",
}
],
reasoning_content=reasoning_content,
)
choices.append(
Choices(message=msg, finish_reason="tool_calls", index=index)
)
reasoning_content = None # flush reasoning content
index += 1
elif isinstance(item, dict):
# Handle raw dict responses (e.g., from GPT-5 Codex)
choice, index = self._handle_raw_dict_response_item(
item=item, index=index
)
if choice is not None:
choices.append(choice)
else:
pass # don't fail request if item in list is not supported
@ -447,9 +517,25 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
self, tools: List[Dict[str, Any]]
) -> List["ALL_RESPONSES_API_TOOL_PARAMS"]:
"""Convert chat completion tools to responses API tools format"""
responses_tools = []
responses_tools: List["ALL_RESPONSES_API_TOOL_PARAMS"] = []
for tool in tools:
responses_tools.append(tool)
# convert function tool from chat completion to responses API format
if tool.get("type") == "function":
function_tool = cast(
ChatCompletionToolParamFunctionChunk, tool.get("function")
)
responses_tools.append(
FunctionToolParam(
name=function_tool["name"],
parameters=function_tool.get("parameters"),
strict=function_tool.get("strict"),
type="function",
description=function_tool.get("description"),
)
)
else:
responses_tools.append(tool) # type: ignore
return cast(List["ALL_RESPONSES_API_TOOL_PARAMS"], responses_tools)
def _map_reasoning_effort(self, reasoning_effort: str) -> Optional[Reasoning]:

38
litellm/constants.py
View file

@ -87,6 +87,35 @@ MAX_TOKEN_TRIMMING_ATTEMPTS = int(
########## Networking constants ##############################################################
_DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client for 1 hour
# Aiohttp connection pooling constants
AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 0))
AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120))
AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300))
# SSL/TLS cipher configuration for faster handshakes
# Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones
# This balances performance with broad compatibility
DEFAULT_SSL_CIPHERS = os.getenv(
"LITELLM_SSL_CIPHERS",
# Priority 1: TLS 1.3 ciphers (fastest, ~50ms handshake)
"TLS_AES_256_GCM_SHA384:" # Fastest observed in testing
"TLS_AES_128_GCM_SHA256:" # Slightly faster than 256-bit
"TLS_CHACHA20_POLY1305_SHA256:" # Fast on ARM/mobile
# Priority 2: TLS 1.2 ECDHE+GCM (fast, ~100ms handshake, widely supported)
"ECDHE-RSA-AES256-GCM-SHA384:"
"ECDHE-RSA-AES128-GCM-SHA256:"
"ECDHE-ECDSA-AES256-GCM-SHA384:"
"ECDHE-ECDSA-AES128-GCM-SHA256:"
# Priority 3: Additional modern ciphers (good balance)
"ECDHE-RSA-CHACHA20-POLY1305:"
"ECDHE-ECDSA-CHACHA20-POLY1305:"
# Priority 4: Widely compatible fallbacks (slower but universally supported)
"ECDHE-RSA-AES256-SHA384:" # Common fallback
"ECDHE-RSA-AES128-SHA256:" # Very widely supported
"AES256-GCM-SHA384:" # Non-PFS fallback (compatibility)
"AES128-GCM-SHA256", # Last resort (maximum compatibility)
)
########### v2 Architecture constants for managing writing updates to the database ###########
REDIS_UPDATE_BUFFER_KEY = "litellm_spend_update_buffer"
REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_spend_update_buffer"
@ -496,6 +525,7 @@ openai_compatible_providers: List = [
"vercel_ai_gateway",
"aiml",
"wandb",
"cometapi",
]
openai_text_completion_compatible_providers: List = (
[ # providers that support `/v1/completions`
@ -820,6 +850,7 @@ BEDROCK_CONVERSE_MODELS = [
"deepseek.v3-v1:0",
"openai.gpt-oss-20b-1:0",
"openai.gpt-oss-120b-1:0",
"anthropic.claude-haiku-4-5-20251001-v1:0",
"anthropic.claude-sonnet-4-5-20250929-v1:0",
"anthropic.claude-opus-4-1-20250805-v1:0",
"anthropic.claude-opus-4-20250514-v1:0",
@ -871,6 +902,7 @@ bedrock_embedding_models: set = set(
"amazon.titan-embed-text-v1",
"cohere.embed-english-v3",
"cohere.embed-multilingual-v3",
"cohere.embed-v4:0",
"twelvelabs.marengo-embed-2-7-v1:0",
]
)
@ -1027,6 +1059,12 @@ PROXY_BATCH_WRITE_AT = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds
DEFAULT_HEALTH_CHECK_INTERVAL = int(
os.getenv("DEFAULT_HEALTH_CHECK_INTERVAL", 300)
) # 5 minutes
DEFAULT_SHARED_HEALTH_CHECK_TTL = int(
os.getenv("DEFAULT_SHARED_HEALTH_CHECK_TTL", 300)
) # 5 minutes - TTL for cached health check results
DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL = int(
os.getenv("DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL", 60)
) # 1 minute - TTL for health check lock
PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS = int(
os.getenv("PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS", 9)
)

173
litellm/experimental_mcp_client/client.py
View file

@ -5,8 +5,9 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers.
import asyncio
import base64
from datetime import timedelta
from typing import Dict, List, Optional, Union
from typing import Callable, Dict, List, Optional, Union
import httpx
from mcp import ClientSession, StdioServerParameters
from mcp.client.sse import sse_client
from mcp.client.stdio import stdio_client
@ -17,6 +18,8 @@ from mcp.types import TextContent
from mcp.types import Tool as MCPTool
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import get_ssl_configuration
from litellm.types.llms.custom_http import VerifyTypes
from litellm.types.mcp import (
MCPAuth,
MCPAuthType,
@ -48,6 +51,7 @@ class MCPClient:
timeout: float = 60.0,
stdio_config: Optional[MCPStdioConfig] = None,
extra_headers: Optional[Dict[str, str]] = None,
ssl_verify: Optional[VerifyTypes] = None,
):
self.server_url: str = server_url
self.transport_type: MCPTransport = transport_type
@ -62,6 +66,7 @@ class MCPClient:
self._task: Optional[asyncio.Task] = None
self.stdio_config: Optional[MCPStdioConfig] = stdio_config
self.extra_headers: Optional[Dict[str, str]] = extra_headers
self.ssl_verify: Optional[VerifyTypes] = ssl_verify
# handle the basic auth value if provided
if auth_value:
self.update_auth_value(auth_value)
@ -81,8 +86,15 @@ class MCPClient:
async def connect(self):
"""Initialize the transport and session."""
if self._session:
verbose_logger.debug(
f"MCP client already connected to {self.server_url or 'stdio'}"
)
return # Already connected
verbose_logger.info(
f"MCP client connecting to {self.server_url or 'stdio'} via {self.transport_type}"
)
try:
if self.transport_type == MCPTransport.stdio:
# For stdio transport, use stdio_client with command-line parameters
@ -102,12 +114,17 @@ class MCPClient:
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
verbose_logger.info(
f"MCP client successfully connected via stdio: {self.stdio_config.get('command', '')}"
)
elif self.transport_type == MCPTransport.sse:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
self._transport_ctx = sse_client(
url=self.server_url,
timeout=self.timeout,
headers=headers,
httpx_client_factory=httpx_client_factory,
)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(
@ -115,15 +132,20 @@ class MCPClient:
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
verbose_logger.info(
f"MCP client successfully connected via SSE to {self.server_url}"
)
else: # http
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
verbose_logger.debug(
"litellm headers for streamablehttp_client: ", headers
"litellm headers for streamablehttp_client: %s", headers
)
self._transport_ctx = streamablehttp_client(
url=self.server_url,
timeout=timedelta(seconds=self.timeout),
headers=headers,
httpx_client_factory=httpx_client_factory,
)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(
@ -131,6 +153,9 @@ class MCPClient:
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
verbose_logger.info(
f"MCP client successfully connected via HTTP to {self.server_url}"
)
except ValueError as e:
# Re-raise ValueError exceptions (like missing stdio_config)
verbose_logger.warning(f"MCP client connection failed: {str(e)}")
@ -150,7 +175,12 @@ class MCPClient:
async def disconnect(self):
"""Clean up session and connections."""
verbose_logger.info(
f"MCP client disconnecting from {self.server_url or 'stdio'}"
)
if self._task and not self._task.done():
verbose_logger.debug("MCP client cancelling background task")
self._task.cancel()
try:
await self._task
@ -159,16 +189,24 @@ class MCPClient:
if self._session:
try:
verbose_logger.debug("MCP client closing session")
await self._session_ctx.__aexit__(None, None, None) # type: ignore
except Exception:
except Exception as e:
verbose_logger.debug(
f"Error closing MCP session: {type(e).__name__}: {str(e)}"
)
pass
self._session = None
self._session_ctx = None
if self._transport_ctx:
try:
verbose_logger.debug("MCP client closing transport")
await self._transport_ctx.__aexit__(None, None, None)
except Exception:
except Exception as e:
verbose_logger.debug(
f"Error closing MCP transport: {type(e).__name__}: {str(e)}"
)
pass
self._transport_ctx = None
self._transport = None
@ -215,27 +253,92 @@ class MCPClient:
return headers
def _create_httpx_client_factory(self) -> Callable[..., httpx.AsyncClient]:
"""
Create a custom httpx client factory that uses LiteLLM's SSL configuration.
This factory follows the same CA bundle path logic as http_handler.py:
1. Check ssl_verify parameter (can be SSLContext, bool, or path to CA bundle)
2. Check SSL_VERIFY environment variable
3. Check SSL_CERT_FILE environment variable
4. Fall back to certifi CA bundle
"""
def factory(
*,
headers: Optional[Dict[str, str]] = None,
timeout: Optional[httpx.Timeout] = None,
auth: Optional[httpx.Auth] = None,
) -> httpx.AsyncClient:
"""Create an httpx.AsyncClient with LiteLLM's SSL configuration."""
# Get unified SSL configuration using the same logic as http_handler.py
ssl_config = get_ssl_configuration(self.ssl_verify)
verbose_logger.debug(
f"MCP client using SSL configuration: {type(ssl_config).__name__}"
)
return httpx.AsyncClient(
headers=headers,
timeout=timeout,
auth=auth,
verify=ssl_config,
follow_redirects=True,
)
return factory
async def list_tools(self) -> List[MCPTool]:
"""List available tools from the server."""
verbose_logger.debug(
f"MCP client listing tools from {self.server_url or 'stdio'}"
)
if not self._session:
verbose_logger.debug("MCP client session not found, attempting to connect")
try:
await self.connect()
except Exception as e:
verbose_logger.warning(f"MCP client connection failed: {str(e)}")
verbose_logger.error(
f"MCP client connection failed during list_tools: {type(e).__name__}: {str(e)}"
)
return []
if self._session is None:
verbose_logger.warning("MCP client session is not initialized")
verbose_logger.error(
"MCP client session is not initialized after connection attempt"
)
return []
try:
result = await self._session.list_tools()
tool_count = len(result.tools)
tool_names = [tool.name for tool in result.tools]
verbose_logger.info(
f"MCP client listed {tool_count} tools from {self.server_url or 'stdio'}: {tool_names}"
)
return result.tools
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_tools was cancelled")
await self.disconnect()
raise
except Exception as e:
verbose_logger.warning(f"MCP client list_tools failed: {str(e)}")
error_type = type(e).__name__
verbose_logger.error(
f"MCP client list_tools failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during list_tools - "
"the MCP server may have crashed, disconnected, or timed out"
)
await self.disconnect()
# Return empty list instead of raising to allow graceful degradation
return []
@ -246,17 +349,28 @@ class MCPClient:
"""
Call an MCP Tool.
"""
verbose_logger.info(
f"MCP client calling tool '{call_tool_request_params.name}' with arguments: {call_tool_request_params.arguments}"
)
if not self._session:
verbose_logger.warning(
"MCP client session not found, attempting to connect"
)
try:
await self.connect()
except Exception as e:
verbose_logger.warning(f"MCP client connection failed: {str(e)}")
verbose_logger.error(
f"MCP client connection failed before tool call: {type(e).__name__}: {str(e)}"
)
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{str(e)}")], isError=True
)
if self._session is None:
verbose_logger.warning("MCP client session is not initialized")
verbose_logger.error(
"MCP client session is not initialized after connection attempt"
)
return MCPCallToolResult(
content=[
TextContent(
@ -266,22 +380,59 @@ class MCPClient:
isError=True,
)
# Check session and transport state before calling tool
verbose_logger.debug(
f"MCP client state before tool call - "
f"session: {'active' if self._session else 'none'}, "
f"transport: {'active' if self._transport else 'none'}, "
f"session_ctx: {'active' if self._session_ctx else 'none'}, "
f"transport_ctx: {'active' if self._transport_ctx else 'none'}"
)
try:
verbose_logger.debug("MCP client sending tool call to session")
tool_result = await self._session.call_tool(
name=call_tool_request_params.name,
arguments=call_tool_request_params.arguments,
)
verbose_logger.info(
f"MCP client tool call '{call_tool_request_params.name}' completed successfully"
)
return tool_result
except asyncio.CancelledError:
verbose_logger.warning("MCP client tool call was cancelled")
await self.disconnect()
raise
except Exception as e:
verbose_logger.warning(f"MCP client call_tool failed: {str(e)}")
import traceback
error_trace = traceback.format_exc()
verbose_logger.debug(f"MCP client tool call traceback:\n{error_trace}")
# Log detailed error information
error_type = type(e).__name__
verbose_logger.error(
f"MCP client call_tool failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Tool: {call_tool_request_params.name}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream - "
"the MCP server may have crashed, disconnected, or timed out. "
"Session and transport will be disconnected."
)
await self.disconnect()
# Return a default error result instead of raising
return MCPCallToolResult(
content=[
TextContent(type="text", text=f"{str(e)}")
TextContent(type="text", text=f"{error_type}: {str(e)}")
], # Empty content for error case
isError=True,
)

9
litellm/files/main.py
View file

@ -18,6 +18,7 @@ from litellm import get_secret_str
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.azure.files.handler import AzureOpenAIFilesAPI
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.openai import FileDeleted, FileObject, OpenAIFilesAPI
from litellm.llms.vertex_ai.files.handler import VertexAIFilesHandler
@ -268,6 +269,7 @@ def create_file(
raise e
@client
async def afile_retrieve(
file_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
@ -308,6 +310,7 @@ async def afile_retrieve(
raise e
@client
def file_retrieve(
file_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
@ -422,6 +425,7 @@ def file_retrieve(
# Delete file
@client
async def afile_delete(
file_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
@ -462,6 +466,7 @@ async def afile_delete(
raise e
@client
def file_delete(
file_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
@ -577,6 +582,7 @@ def file_delete(
# List files
@client
async def afile_list(
custom_llm_provider: Literal["openai", "azure"] = "openai",
purpose: Optional[str] = None,
@ -617,6 +623,7 @@ async def afile_list(
raise e
@client
def file_list(
custom_llm_provider: Literal["openai", "azure"] = "openai",
purpose: Optional[str] = None,
@ -729,6 +736,7 @@ def file_list(
raise e
@client
async def afile_content(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
@ -771,6 +779,7 @@ async def afile_content(
raise e
@client
def file_content(
file_id: str,
model: Optional[str] = None,

2
litellm/google_genai/main.py
View file

@ -405,6 +405,7 @@ async def agenerate_content_stream(
config=setup_result.generate_content_config_dict,
litellm_params=setup_result.litellm_params,
tools=tools,
stream=True,
**kwargs,
)
)
@ -485,6 +486,7 @@ def generate_content_stream(
config=setup_result.generate_content_config_dict,
_is_async=_is_async,
litellm_params=setup_result.litellm_params,
stream=True,
**kwargs,
)

4
litellm/integrations/opentelemetry.py
View file

@ -576,7 +576,7 @@ class OpenTelemetry(CustomLogger):
return
litellm_params = kwargs.get("litellm_params", {})
metadata = litellm_params.get("metadata", {})
metadata = litellm_params.get("metadata") or {}
generation_name = metadata.get("generation_name")
raw_span_name = generation_name if generation_name else RAW_REQUEST_SPAN_NAME
@ -1178,7 +1178,7 @@ class OpenTelemetry(CustomLogger):
def _get_span_name(self, kwargs):
litellm_params = kwargs.get("litellm_params", {})
metadata = litellm_params.get("metadata", {})
metadata = litellm_params.get("metadata") or {}
generation_name = metadata.get("generation_name")
if generation_name:

102
litellm/integrations/posthog.py
View file

@ -11,11 +11,10 @@ For batching specific details see CustomBatchLogger class
import asyncio
import os
from litellm._uuid import uuid
from typing import Any, Dict, Optional
from typing import Any, Dict, Optional, Tuple
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
@ -26,7 +25,7 @@ from litellm.types.integrations.posthog import (
POSTHOG_MAX_BATCH_SIZE,
PostHogEventPayload,
)
from litellm.types.utils import StandardLoggingPayload
from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload
class PostHogLogger(CustomBatchLogger):
@ -72,17 +71,21 @@ class PostHogLogger(CustomBatchLogger):
verbose_logger.debug(
"PostHog: Sync logging - Enters logging function for model %s", kwargs
)
api_key, api_url = self._get_credentials_for_request(kwargs)
if api_key is None or api_url is None:
raise Exception("PostHog credentials not found in kwargs")
event_payload = self.create_posthog_event_payload(kwargs)
headers = {
"Content-Type": "application/json",
}
payload = self._create_posthog_payload([event_payload])
payload = self._create_posthog_payload([event_payload], api_key)
capture_url = f"{api_url.rstrip('/')}/batch/"
response = self.sync_client.post(
url=self.capture_url,
url=capture_url,
json=payload,
headers=headers,
)
@ -92,9 +95,9 @@ class PostHogLogger(CustomBatchLogger):
raise Exception(
f"Response from PostHog API status_code: {response.status_code}, text: {response.text}"
)
verbose_logger.debug("PostHog: Sync event successfully sent")
except Exception as e:
verbose_logger.exception(f"PostHog Sync Layer Error - {str(e)}")
@ -122,9 +125,15 @@ class PostHogLogger(CustomBatchLogger):
async def _log_async_event(self, kwargs, response_obj=None, start_time=0.0, end_time=0.0):
# Note: response_obj, start_time, end_time not used - all data comes from kwargs
api_key, api_url = self._get_credentials_for_request(kwargs)
event_payload = self.create_posthog_event_payload(kwargs)
self.log_queue.append(event_payload)
# Store event with its credentials for batch sending
self.log_queue.append({
"event": event_payload,
"api_key": api_key,
"api_url": api_url
})
verbose_logger.debug(
f"PostHog, event added to queue. Will flush in {self.flush_interval} seconds..."
)
@ -257,16 +266,42 @@ class PostHogLogger(CustomBatchLogger):
metadata = self._extract_metadata(kwargs)
user_id = self._safe_get(metadata, "user_id")
if user_id:
return str(user_id)
return str(user_id)
end_user = self._safe_get(standard_logging_object, "end_user")
if end_user:
return str(end_user)
trace_id = self._safe_get(standard_logging_object, "trace_id")
if trace_id:
return str(trace_id)
return str(trace_id)
return self._safe_uuid()
def _get_credentials_for_request(self, kwargs: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
"""
Get PostHog credentials for this request.
Checks for per-request credentials in standard_callback_dynamic_params,
falls back to instance defaults from environment variables.
Args:
kwargs: Request kwargs containing standard_callback_dynamic_params
Returns:
tuple[str, str]: (api_key, api_url)
"""
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
if standard_callback_dynamic_params is not None:
api_key = standard_callback_dynamic_params.get("posthog_api_key") or self.POSTHOG_API_KEY
api_url = standard_callback_dynamic_params.get("posthog_api_url") or self.posthog_host
else:
api_key = self.POSTHOG_API_KEY
api_url = self.posthog_host
return api_key, api_url
async def async_send_batch(self):
"""
Sends the in memory logs queue to PostHog API
@ -282,23 +317,34 @@ class PostHogLogger(CustomBatchLogger):
f"PostHog: Sending batch of {len(self.log_queue)} events"
)
headers = {
"Content-Type": "application/json",
}
# Group events by credentials for batch sending
batches_by_credentials: Dict[tuple[str, str], list] = {}
for item in self.log_queue:
key = (item["api_key"], item["api_url"])
if key not in batches_by_credentials:
batches_by_credentials[key] = []
batches_by_credentials[key].append(item["event"])
payload = self._create_posthog_payload(list(self.log_queue))
# Send each batch to its respective PostHog instance
for (api_key, api_url), events in batches_by_credentials.items():
headers = {
"Content-Type": "application/json",
}
response = await self.async_client.post(
url=self.capture_url,
json=payload,
headers=headers,
)
response.raise_for_status()
payload = self._create_posthog_payload(events, api_key)
capture_url = f"{api_url.rstrip('/')}/batch/"
if response.status_code != 200:
raise Exception(
f"Response from PostHog API status_code: {response.status_code}, text: {response.text}"
response = await self.async_client.post(
url=capture_url,
json=payload,
headers=headers,
)
response.raise_for_status()
if response.status_code != 200:
raise Exception(
f"Response from PostHog API status_code: {response.status_code}, text: {response.text}"
)
verbose_logger.debug(
f"PostHog: Batch of {len(self.log_queue)} events successfully sent"
@ -324,8 +370,8 @@ class PostHogLogger(CustomBatchLogger):
def _safe_uuid(self) -> str:
return str(uuid.uuid4())
def _create_posthog_payload(self, events: list) -> Dict[str, Any]:
return {"api_key": self.POSTHOG_API_KEY, "batch": events}
def _create_posthog_payload(self, events: list, api_key: str) -> Dict[str, Any]:
return {"api_key": api_key, "batch": events}
def _safe_get(self, obj: Any, key: str, default: Any = None) -> Any:
if obj is None or not hasattr(obj, 'get'):

4
litellm/litellm_core_utils/litellm_logging.py
View file

@ -81,12 +81,12 @@ from litellm.types.llms.openai import (
)
from litellm.types.mcp import MCPPostCallResponseObject
from litellm.types.rerank import RerankResponse
from litellm.types.router import CustomPricingLiteLLMParams
from litellm.types.utils import (
CachingDetails,
CallTypes,
CostBreakdown,
CostResponseTypes,
CustomPricingLiteLLMParams,
DynamicPromptManagementParamLiteral,
EmbeddingResponse,
GuardrailStatus,
@ -4040,6 +4040,7 @@ class StandardLoggingPayloadSetup:
usage_object=usage_object,
requester_custom_headers=None,
cold_storage_object_key=None,
user_api_key_auth_metadata=None,
)
if isinstance(metadata, dict):
# Filter the metadata dictionary to include only the specified keys
@ -4755,6 +4756,7 @@ def get_standard_logging_metadata(
requester_custom_headers=None,
user_api_key_request_route=None,
cold_storage_object_key=None,
user_api_key_auth_metadata=None,
)
if isinstance(metadata, dict):
# Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields

9
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -693,6 +693,15 @@ class CostCalculatorUtils:
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value:
from litellm.llms.cometapi.image_generation.cost_calculator import (
cost_calculator as cometapi_image_cost_calculator,
)
return cometapi_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
from litellm.llms.gemini.image_generation.cost_calculator import (
cost_calculator as gemini_image_cost_calculator,

4
litellm/litellm_core_utils/model_response_utils.py
View file

@ -84,7 +84,9 @@ def _has_meaningful_content(value: Any) -> bool:
return False
if isinstance(value, str):
return len(value.strip()) > 0
# Don't strip whitespace - preserve all content including newlines, spaces, etc.
# Even pure whitespace characters like '\n' or ' ' are meaningful content
return len(value) > 0
if isinstance(value, (list, dict)):
return len(value) > 0

290
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -2,7 +2,6 @@ import copy
import json
import mimetypes
import re
from litellm._uuid import uuid
import xml.etree.ElementTree as ET
from enum import Enum
from typing import Any, List, Optional, Tuple, cast, overload
@ -13,6 +12,7 @@ import litellm
import litellm.types
import litellm.types.llms
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client
from litellm.types.files import get_file_extension_from_mime_type
from litellm.types.llms.anthropic import *
@ -232,7 +232,6 @@ def ollama_pt(
## MERGE CONSECUTIVE ASSISTANT CONTENT ##
while msg_i < len(messages) and messages[msg_i]["role"] == "assistant":
assistant_content_str += convert_content_list_to_str(messages[msg_i])
msg_i += 1
tool_calls = messages[msg_i].get("tool_calls")
ollama_tool_calls = []
@ -258,7 +257,7 @@ def ollama_pt(
f"Tool Calls: {json.dumps(ollama_tool_calls, indent=2)}"
)
msg_i += 1
msg_i += 1
if assistant_content_str:
prompt += f"### Assistant:\n{assistant_content_str}\n\n"
@ -365,62 +364,20 @@ def phind_codellama_pt(messages):
return prompt
def hf_chat_template( # noqa: PLR0915
model: str, messages: list, chat_template: Optional[Any] = None
):
# Define Jinja2 environment
env = ImmutableSandboxedEnvironment()
def raise_exception(message):
raise Exception(f"Error message - {message}")
# Create a template object from the template text
env.globals["raise_exception"] = raise_exception
## get the tokenizer config from huggingface
bos_token = ""
eos_token = ""
if chat_template is None:
def _get_tokenizer_config(hf_model_name):
try:
url = f"https://huggingface.co/{hf_model_name}/raw/main/tokenizer_config.json"
# Make a GET request to fetch the JSON data
client = HTTPHandler(concurrent_limit=1)
response = client.get(url)
except Exception as e:
raise e
if response.status_code == 200:
# Parse the JSON data
tokenizer_config = json.loads(response.content)
return {"status": "success", "tokenizer": tokenizer_config}
else:
return {"status": "failure"}
if model in litellm.known_tokenizer_config:
tokenizer_config = litellm.known_tokenizer_config[model]
else:
tokenizer_config = _get_tokenizer_config(model)
litellm.known_tokenizer_config.update({model: tokenizer_config})
if (
tokenizer_config["status"] == "failure"
or "chat_template" not in tokenizer_config["tokenizer"]
):
raise Exception("No chat template found")
## read the bos token, eos token and chat template from the json
tokenizer_config = tokenizer_config["tokenizer"] # type: ignore
bos_token = tokenizer_config["bos_token"] # type: ignore
if bos_token is not None and not isinstance(bos_token, str):
if isinstance(bos_token, dict):
bos_token = bos_token.get("content", None)
eos_token = tokenizer_config["eos_token"] # type: ignore
if eos_token is not None and not isinstance(eos_token, str):
if isinstance(eos_token, dict):
eos_token = eos_token.get("content", None)
chat_template = tokenizer_config["chat_template"] # type: ignore
def _render_chat_template(env, chat_template: str, bos_token: str, eos_token: str, messages: list) -> str:
"""
Shared template rendering logic for both sync and async hf_chat_template
Args:
env: Jinja2 environment
chat_template: Chat template string
bos_token: Beginning of sequence token
eos_token: End of sequence token
messages: Messages to render
Returns:
Rendered template string
"""
try:
template = env.from_string(chat_template) # type: ignore
except Exception as e:
@ -435,7 +392,6 @@ def hf_chat_template( # noqa: PLR0915
bos_token="<bos>",
)
return True
# This will be raised if Jinja attempts to render the system message and it can't
except Exception:
return False
@ -469,7 +425,7 @@ def hf_chat_template( # noqa: PLR0915
)
except Exception as e:
if "Conversation roles must alternate user/assistant" in str(e):
# reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, add a blank 'user' or 'assistant' message to ensure compatibility
# reformat messages to ensure user/assistant are alternating
new_messages = []
for i in range(len(reformatted_messages) - 1):
new_messages.append(reformatted_messages[i])
@ -495,6 +451,188 @@ def hf_chat_template( # noqa: PLR0915
) # don't use verbose_logger.exception, if exception is raised
async def _afetch_and_extract_template(
model: str, chat_template: Optional[Any], get_config_fn, get_template_fn
) -> Tuple[str, str, str]:
"""
Async version: Fetch template and tokens from HuggingFace.
Returns: (chat_template, bos_token, eos_token)
"""
from litellm.litellm_core_utils.prompt_templates.huggingface_template_handler import (
_extract_token_value,
)
bos_token = ""
eos_token = ""
if chat_template is None:
# Fetch or retrieve cached tokenizer config
if model in litellm.known_tokenizer_config:
tokenizer_config = litellm.known_tokenizer_config[model]
else:
tokenizer_config = await get_config_fn(hf_model_name=model)
litellm.known_tokenizer_config.update({model: tokenizer_config})
# Try to get chat template from tokenizer_config.json first
if (
tokenizer_config.get("status") == "success"
and "tokenizer" in tokenizer_config
and isinstance(tokenizer_config["tokenizer"], dict)
and "chat_template" in tokenizer_config["tokenizer"]
):
tokenizer_data: dict = tokenizer_config["tokenizer"] # type: ignore
bos_token = _extract_token_value(
token_value=tokenizer_data.get("bos_token")
)
eos_token = _extract_token_value(
token_value=tokenizer_data.get("eos_token")
)
chat_template = tokenizer_data["chat_template"]
else:
# Fallback: Try to fetch chat template from separate .jinja file
template_result = await get_template_fn(hf_model_name=model)
if template_result.get("status") == "success":
chat_template = template_result["chat_template"]
# Still try to get tokens from tokenizer_config if available
if (
tokenizer_config.get("status") == "success"
and "tokenizer" in tokenizer_config
and isinstance(tokenizer_config["tokenizer"], dict)
):
tokenizer_data: dict = tokenizer_config["tokenizer"] # type: ignore
bos_token = _extract_token_value(
token_value=tokenizer_data.get("bos_token")
)
eos_token = _extract_token_value(
token_value=tokenizer_data.get("eos_token")
)
else:
raise Exception("No chat template found")
return chat_template, bos_token, eos_token # type: ignore
def _fetch_and_extract_template(
model: str, chat_template: Optional[Any], get_config_fn, get_template_fn
) -> Tuple[str, str, str]:
"""
Sync version: Fetch template and tokens from HuggingFace.
Returns: (chat_template, bos_token, eos_token)
"""
from litellm.litellm_core_utils.prompt_templates.huggingface_template_handler import (
_extract_token_value,
)
bos_token = ""
eos_token = ""
if chat_template is None:
# Fetch or retrieve cached tokenizer config
if model in litellm.known_tokenizer_config:
tokenizer_config = litellm.known_tokenizer_config[model]
else:
tokenizer_config = get_config_fn(hf_model_name=model)
litellm.known_tokenizer_config.update({model: tokenizer_config})
# Try to get chat template from tokenizer_config.json first
if (
tokenizer_config.get("status") == "success"
and "tokenizer" in tokenizer_config
and isinstance(tokenizer_config["tokenizer"], dict)
and "chat_template" in tokenizer_config["tokenizer"]
):
tokenizer_data: dict = tokenizer_config["tokenizer"] # type: ignore
bos_token = _extract_token_value(
token_value=tokenizer_data.get("bos_token")
)
eos_token = _extract_token_value(
token_value=tokenizer_data.get("eos_token")
)
chat_template = tokenizer_data["chat_template"]
else:
# Fallback: Try to fetch chat template from separate .jinja file
template_result = get_template_fn(hf_model_name=model)
if template_result.get("status") == "success":
chat_template = template_result["chat_template"]
# Still try to get tokens from tokenizer_config if available
if (
tokenizer_config.get("status") == "success"
and "tokenizer" in tokenizer_config
and isinstance(tokenizer_config["tokenizer"], dict)
):
tokenizer_data: dict = tokenizer_config["tokenizer"] # type: ignore
bos_token = _extract_token_value(
token_value=tokenizer_data.get("bos_token")
)
eos_token = _extract_token_value(
token_value=tokenizer_data.get("eos_token")
)
else:
raise Exception("No chat template found")
return chat_template, bos_token, eos_token # type: ignore
async def ahf_chat_template(
model: str, messages: list, chat_template: Optional[Any] = None
):
"""HuggingFace chat template (async version)"""
from litellm.litellm_core_utils.prompt_templates.huggingface_template_handler import (
_aget_chat_template_file,
_aget_tokenizer_config,
strftime_now,
)
env = ImmutableSandboxedEnvironment()
env.globals["raise_exception"] = lambda msg: Exception(f"Error message - {msg}")
env.globals["strftime_now"] = strftime_now
template, bos_token, eos_token = await _afetch_and_extract_template(
model=model,
chat_template=chat_template,
get_config_fn=_aget_tokenizer_config,
get_template_fn=_aget_chat_template_file,
)
return _render_chat_template(
env=env,
chat_template=template,
bos_token=bos_token,
eos_token=eos_token,
messages=messages,
)
def hf_chat_template(
model: str, messages: list, chat_template: Optional[Any] = None
):
"""HuggingFace chat template (sync version)"""
from litellm.litellm_core_utils.prompt_templates.huggingface_template_handler import (
_get_chat_template_file,
_get_tokenizer_config,
strftime_now,
)
env = ImmutableSandboxedEnvironment()
env.globals["raise_exception"] = lambda msg: Exception(f"Error message - {msg}")
env.globals["strftime_now"] = strftime_now
template, bos_token, eos_token = _fetch_and_extract_template(
model=model,
chat_template=chat_template,
get_config_fn=_get_tokenizer_config,
get_template_fn=_get_chat_template_file,
)
return _render_chat_template(
env=env,
chat_template=template,
bos_token=bos_token,
eos_token=eos_token,
messages=messages,
)
def deepseek_r1_pt(messages):
return hf_chat_template(
model="deepseek-r1/deepseek-r1-7b-instruct", messages=messages
@ -4032,33 +4170,9 @@ def prompt_factory(
elif custom_llm_provider == "azure_text":
return azure_text_pt(messages=messages)
elif custom_llm_provider == "watsonx":
if "granite" in model and "chat" in model:
# granite-13b-chat-v1 and granite-13b-chat-v2 use a specific prompt template
return ibm_granite_pt(messages=messages)
elif "ibm-mistral" in model and "instruct" in model:
# models like ibm-mistral/mixtral-8x7b-instruct-v01-q use the mistral instruct prompt template
return mistral_instruct_pt(messages=messages)
elif "meta-llama/llama-3" in model and "instruct" in model:
# https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3/
return custom_prompt(
role_dict={
"system": {
"pre_message": "<|start_header_id|>system<|end_header_id|>\n",
"post_message": "<|eot_id|>",
},
"user": {
"pre_message": "<|start_header_id|>user<|end_header_id|>\n",
"post_message": "<|eot_id|>",
},
"assistant": {
"pre_message": "<|start_header_id|>assistant<|end_header_id|>\n",
"post_message": "<|eot_id|>",
},
},
messages=messages,
initial_prompt_value="<|begin_of_text|>",
final_prompt_value="<|start_header_id|>assistant<|end_header_id|>\n",
)
from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig
return IBMWatsonXChatConfig.apply_prompt_template(model=model, messages=messages)
try:
if "meta-llama/llama-2" in model and "chat" in model:
return llama_2_chat_pt(messages=messages)

Some files were not shown because too many files have changed in this diff Show more