scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 154

Merge branch 'BerriAI:main' into main

Browse source
This commit is contained in:
Gauri Nagavkar 2026-02-15 07:30:40 -08:00 • committed by GitHub
commit 15d8367bd0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
918 changed files with 45598 additions and 14553 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

101
.circleci/config.yml
View file

@ -1656,7 +1656,7 @@ jobs:
- search_coverage.xml
- search_coverage
# Split litellm_mapped_tests into 3 parallel jobs for 3x faster execution
litellm_mapped_tests_proxy:
litellm_mapped_tests_proxy_part1:
docker:
- image: cimg/python:3.11
auth:
@ -1667,23 +1667,53 @@ jobs:
steps:
- setup_litellm_test_deps
- run:
name: Run proxy tests
name: Run proxy tests part 1 (high-volume directories)
command: |
prisma generate
python -m pytest tests/test_litellm/proxy --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
export PYTHONUNBUFFERED=1
python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
no_output_timeout: 60m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_proxy_tests_coverage.xml
mv .coverage litellm_proxy_tests_coverage
mv coverage.xml litellm_proxy_tests_part1_coverage.xml
mv .coverage litellm_proxy_tests_part1_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_proxy_tests_coverage.xml
- litellm_proxy_tests_coverage
- litellm_proxy_tests_part1_coverage.xml
- litellm_proxy_tests_part1_coverage
litellm_mapped_tests_proxy_part2:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: xlarge
steps:
- setup_litellm_test_deps
- run:
name: Run proxy tests part 2 (all other tests)
command: |
prisma generate
export PYTHONUNBUFFERED=1
python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
no_output_timeout: 60m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_proxy_tests_part2_coverage.xml
mv .coverage litellm_proxy_tests_part2_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_proxy_tests_part2_coverage.xml
- litellm_proxy_tests_part2_coverage
litellm_mapped_tests_llms:
docker:
- image: cimg/python:3.11
@ -1724,7 +1754,7 @@ jobs:
- run:
name: Run core tests
command: |
python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --ignore=tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -1765,6 +1795,33 @@ jobs:
paths:
- litellm_core_utils_tests_coverage.xml
- litellm_core_utils_tests_coverage
litellm_mapped_tests_mcps:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: xlarge
steps:
- setup_litellm_test_deps
- run:
name: Run MCP client tests
command: |
python -m pytest tests/test_litellm/experimental_mcp_client --cov=litellm --cov-report=xml --junitxml=test-results/junit-mcps.xml --durations=10 -n 4 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_mcps_tests_coverage.xml
mv .coverage litellm_mcps_tests_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_mcps_tests_coverage.xml
- litellm_mcps_tests_coverage
litellm_mapped_tests_integrations:
docker:
- image: cimg/python:3.11
@ -3597,9 +3654,11 @@ jobs:
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME="us-east-1" \
-e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
@ -3652,7 +3711,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
coverage combine llm_translation_coverage realtime_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage combine llm_translation_coverage realtime_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@ -4392,7 +4451,13 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_proxy:
- litellm_mapped_tests_proxy_part1:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_proxy_part2:
filters:
branches:
only:
@ -4410,6 +4475,12 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_mcps:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_integrations:
filters:
branches:
@ -4469,9 +4540,11 @@ workflows:
- llm_responses_api_testing
- ocr_testing
- search_testing
- litellm_mapped_tests_proxy
- litellm_mapped_tests_proxy_part1
- litellm_mapped_tests_proxy_part2
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
- litellm_mapped_tests_mcps
- litellm_mapped_tests_integrations
- litellm_mapped_tests_litellm_core_utils
- litellm_mapped_enterprise_tests
@ -4548,9 +4621,11 @@ workflows:
- llm_responses_api_testing
- ocr_testing
- search_testing
- litellm_mapped_tests_proxy
- litellm_mapped_tests_proxy_part1
- litellm_mapped_tests_proxy_part2
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
- litellm_mapped_tests_mcps
- litellm_mapped_tests_integrations
- litellm_mapped_tests_litellm_core_utils
- litellm_mapped_enterprise_tests

19
.github/actions/helm-oci-chart-releaser/action.yml vendored
View file

@ -40,38 +40,33 @@ outputs:
runs:
using: composite
steps:
- name: Helm | Setup
uses: azure/setup-helm@v4
with:
version: v3.20.0
- name: Helm | Login
shell: bash
run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }}
env:
HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Dependency
if: inputs.update_dependencies == 'true'
shell: bash
run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
env:
HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Package
shell: bash
run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }}
env:
HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Push
shell: bash
run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }}
env:
HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Logout
shell: bash
run: helm registry logout ${{ inputs.registry }}
env:
HELM_EXPERIMENTAL_OCI: '1'
- name: Helm | Output
id: output
shell: bash
run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT

14
.github/workflows/test-litellm-matrix.yml vendored
View file

@ -93,17 +93,3 @@ jobs:
--maxfail=10 \
-n ${{ matrix.test-group.workers }} \
--durations=20
# Aggregate job to require all matrix jobs pass
test-complete:
needs: test
runs-on: ubuntu-latest
if: always()
steps:
- name: Check test results
run: |
if [ "${{ needs.test.result }}" != "success" ]; then
echo "Some test groups failed"
exit 1
fi
echo "All test groups passed!"

32
.github/workflows/test-litellm-ui-build.yml vendored Normal file
View file

@ -0,0 +1,32 @@
name: UI Build Check
permissions:
contents: read
on:
pull_request:
branches: [main]
jobs:
build-ui:
runs-on: ubuntu-latest
timeout-minutes: 10
defaults:
run:
working-directory: ui/litellm-dashboard
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: "20"
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
run: npm install
- name: Build
run: npm run build

1
.gitignore vendored
View file

@ -2,6 +2,7 @@
.venv
.venv_policy_test
.env
.claude
.newenv
newenv/*
litellm/proxy/myenv/*

52
.semgrep/rules/README.md
View file

@ -1,52 +1,22 @@
# Custom Semgrep Rules
# Custom Semgrep rules for LiteLLM
All `.yml` files under `.semgrep/rules/` run in CI (CircleCI `semgrep` job).
Add custom rule YAML files here. Semgrep loads all `.yml`/`.yaml` files under this directory.
## Add a Rule
* Add a `.yml` file under `.semgrep/rules/<language>/<domain>/`
[Rule syntax →](https://semgrep.dev/docs/writing-rules/rule-syntax/)
## Organizing Rules
### Structure: language → domain
```
.semgrep/rules/<language>/<domain>/<rule-name>.yml
```
Examples:
- `python/security/unsafe-yaml-load.yml`
- `python/reliability/missing-timeout-http.yml`
- `python/performance/blocking-io-in-async.yml`
### Rule metadata
Match tags to the folder for consistent filtering:
```yaml
metadata:
tags: [python, security]
```
### Severity expectations
All rules must fail CI on findings. No warn-only rules.
- Use `severity: ERROR` in rule metadata
- If a rule is noisy → refine until low false positives before adding
## Run Locally
**Run only custom rules (CI / fail on findings):**
```bash
semgrep scan --config .semgrep/rules . --error
```
With Semgrep registry:
**Run with registry + custom rules:**
```bash
semgrep scan --config auto --config .semgrep/rules .
```
**Layout:**
- `python/` – Python-specific rules (security, patterns)
- Add more subdirs as needed (e.g. `generic/` for language-agnostic rules)
See [Semgrep rule syntax](https://semgrep.dev/docs/writing-rules/rule-syntax/).

14
.semgrep/rules/python/unbounded-memory.yml Normal file
View file

@ -0,0 +1,14 @@
# Unbounded memory growth – data structures without a clear max limit
# Can lead to OOM under load.
rules:
- id: unbounded-asyncio-queue
message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues).
severity: ERROR
languages: [python]
pattern-either:
- pattern: asyncio.Queue()
- pattern: asyncio.Queue(maxsize=0)
metadata:
category: correctness
cwe: "CWE-400: Uncontrolled Resource Consumption"

4
deploy/charts/litellm-helm/Chart.yaml
View file

@ -26,6 +26,10 @@ version: 1.1.0
# It is recommended to use it with quotes.
appVersion: v1.80.12
annotations:
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
org.opencontainers.image.url: "https://docs.litellm.ai/"
dependencies:
- name: "postgresql"
version: ">=13.3.0"

19
docs/my-website/blog/claude_opus_4_6/index.md
View file

@ -389,6 +389,10 @@ Compaction blocks are also supported in streaming mode. You'll receive:
### Adaptive Thinking
:::note
When using `reasoning_effort` with Claude Opus 4.6, all values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets with `type: "enabled"`, pass the native `thinking` parameter directly (see "Native thinking param" tab below).
:::
<Tabs>
<TabItem value="completions" label="/chat/completions">
@ -434,6 +438,21 @@ curl --location 'http://0.0.0.0:4000/v1/messages' \
}'
```
</TabItem>
<TabItem value="native" label="Native thinking param">
Use the `thinking` parameter directly for adaptive thinking via the SDK:
```python
import litellm
response = litellm.completion(
model="anthropic/claude-opus-4-6",
messages=[{"role": "user", "content": "Solve this complex problem: What is the optimal strategy for..."}],
thinking={"type": "adaptive"},
)
```
</TabItem>
</Tabs>

394
docs/my-website/blog/minimax_m2_5/index.md Normal file
View file

@ -0,0 +1,394 @@
---
slug: minimax_m2_5
title: "Day 0 Support: MiniMax-M2.5"
date: 2026-02-12T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Day 0 support for MiniMax-M2.5 on LiteLLM"
tags: [minimax, M2.5, llm]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway.
## Supported Models
LiteLLM supports the following MiniMax models:
| Model | Description | Input Cost | Output Cost | Context Window |
|-------|-------------|------------|-------------|----------------|
| **MiniMax-M2.5** | Advanced reasoning, Agentic capabilities | $0.3/M tokens | $1.2/M tokens | 1M tokens |
| **MiniMax-M2.5-lightning** | Faster and More Agile (~100 tps) | $0.3/M tokens | $2.4/M tokens | 1M tokens |
## Features Supported
- **Prompt Caching**: Reduce costs with cached prompts ($0.03/M tokens for cache read, $0.375/M tokens for cache write)
- **Function Calling**: Built-in tool calling support
- **Reasoning**: Advanced reasoning capabilities with thinking support
- **System Messages**: Full system message support
- **Cost Tracking**: Automatic cost calculation for all requests
## Docker Image
```bash
docker pull litellm/litellm:v1.81.3-stable
```
## Usage - OpenAI Compatible API (/v1/chat/completions)
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: minimax-m2-5
litellm_params:
model: minimax/MiniMax-M2.5
api_key: os.environ/MINIMAX_API_KEY
api_base: https://api.minimax.io/v1
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e MINIMAX_API_KEY=$MINIMAX_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.3-stable \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "minimax-m2-5",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>
### With Reasoning Split
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "minimax-m2-5",
"messages": [
{
"role": "user",
"content": "Solve: 2+2=?"
}
],
"extra_body": {
"reasoning_split": true
}
}'
```
## Usage - Anthropic Compatible API (/v1/messages)
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: minimax-m2-5
litellm_params:
model: minimax/MiniMax-M2.5
api_key: os.environ/MINIMAX_API_KEY
api_base: https://api.minimax.io/anthropic/v1/messages
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e MINIMAX_API_KEY=$MINIMAX_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.3-stable \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "minimax-m2-5",
"max_tokens": 1000,
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>
### With Thinking
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "minimax-m2-5",
"max_tokens": 1000,
"thinking": {
"type": "enabled",
"budget_tokens": 1000
},
"messages": [
{
"role": "user",
"content": "Solve: 2+2=?"
}
]
}'
```
## Usage - LiteLLM SDK
### OpenAI-compatible API
```python
import litellm
response = litellm.completion(
model="minimax/MiniMax-M2.5",
messages=[
{"role": "user", "content": "Hello, how are you?"}
],
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
print(response.choices[0].message.content)
```
### Anthropic-compatible API
```python
import litellm
response = litellm.anthropic.messages.acreate(
model="minimax/MiniMax-M2.5",
messages=[{"role": "user", "content": "Hello, how are you?"}],
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/anthropic/v1/messages",
max_tokens=1000
)
print(response.choices[0].message.content)
```
### With Thinking
```python
response = litellm.anthropic.messages.acreate(
model="minimax/MiniMax-M2.5",
messages=[{"role": "user", "content": "Solve: 2+2=?"}],
thinking={"type": "enabled", "budget_tokens": 1000},
api_key="your-minimax-api-key"
)
# Access thinking content
for block in response.choices[0].message.content:
if hasattr(block, 'type') and block.type == 'thinking':
print(f"Thinking: {block.thinking}")
```
### With Reasoning Split (OpenAI API)
```python
response = litellm.completion(
model="minimax/MiniMax-M2.5",
messages=[
{"role": "user", "content": "Solve: 2+2=?"}
],
extra_body={"reasoning_split": True},
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
# Access thinking and response
if hasattr(response.choices[0].message, 'reasoning_details'):
print(f"Thinking: {response.choices[0].message.reasoning_details}")
print(f"Response: {response.choices[0].message.content}")
```
## Cost Tracking
LiteLLM automatically tracks costs for MiniMax-M2.5 requests. The pricing is:
- **Input**: $0.3 per 1M tokens
- **Output**: $1.2 per 1M tokens
- **Cache Read**: $0.03 per 1M tokens
- **Cache Write**: $0.375 per 1M tokens
### Accessing Cost Information
```python
response = litellm.completion(
model="minimax/MiniMax-M2.5",
messages=[{"role": "user", "content": "Hello!"}],
api_key="your-minimax-api-key"
)
# Access cost information
print(f"Cost: ${response._hidden_params.get('response_cost', 0)}")
```
## Streaming Support
### OpenAI API
```python
response = litellm.completion(
model="minimax/MiniMax-M2.5",
messages=[{"role": "user", "content": "Tell me a story"}],
stream=True,
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
```
### Streaming with Reasoning Split
```python
stream = litellm.completion(
model="minimax/MiniMax-M2.5",
messages=[
{"role": "user", "content": "Tell me a story"},
],
extra_body={"reasoning_split": True},
stream=True,
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
reasoning_buffer = ""
text_buffer = ""
for chunk in stream:
if hasattr(chunk.choices[0].delta, "reasoning_details") and chunk.choices[0].delta.reasoning_details:
for detail in chunk.choices[0].delta.reasoning_details:
if "text" in detail:
reasoning_text = detail["text"]
new_reasoning = reasoning_text[len(reasoning_buffer):]
if new_reasoning:
print(new_reasoning, end="", flush=True)
reasoning_buffer = reasoning_text
if chunk.choices[0].delta.content:
content_text = chunk.choices[0].delta.content
new_text = content_text[len(text_buffer):] if text_buffer else content_text
if new_text:
print(new_text, end="", flush=True)
text_buffer = content_text
```
## Using with Native SDKs
### Anthropic SDK via LiteLLM Proxy
```python
import os
os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM proxy key
import anthropic
client = anthropic.Anthropic()
message = client.messages.create(
model="minimax-m2-5",
max_tokens=1000,
system="You are a helpful assistant.",
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "Hi, how are you?"
}
]
}
]
)
for block in message.content:
if block.type == "thinking":
print(f"Thinking:\n{block.thinking}\n")
elif block.type == "text":
print(f"Text:\n{block.text}\n")
```
### OpenAI SDK via LiteLLM Proxy
```python
import os
os.environ["OPENAI_BASE_URL"] = "http://localhost:4000"
os.environ["OPENAI_API_KEY"] = "sk-1234" # Your LiteLLM proxy key
from openai import OpenAI
client = OpenAI()
response = client.chat.completions.create(
model="minimax-m2-5",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hi, how are you?"},
],
extra_body={"reasoning_split": True},
)
# Access thinking and response
if hasattr(response.choices[0].message, 'reasoning_details'):
print(f"Thinking:\n{response.choices[0].message.reasoning_details[0]['text']}\n")
print(f"Text:\n{response.choices[0].message.content}\n")
```

6
docs/my-website/docs/adding_provider/generic_guardrail_api.md
View file

@ -93,6 +93,12 @@ Implement `POST /beta/litellm_basic_guardrail_api`
"user_api_key_end_user_id": "end user id associated with the litellm virtual key used",
"user_api_key_org_id": "org id associated with the litellm virtual key used"
},
"request_headers": { // optional: inbound request headers (allowlist). Allowed headers show their value; all others show "[present]" to indicate the header existed.
"User-Agent": "OpenAI/Python 2.17.0",
"Content-Type": "application/json",
"X-Request-Id": "[present]"
},
"litellm_version": "1.x.y", // optional: LiteLLM library version running this proxy
"input_type": "request", // "request" or "response"
"litellm_call_id": "unique_call_id", // the call id of the individual LLM call
"litellm_trace_id": "trace_id", // the trace id of the LLM call - useful if there are multiple LLM calls for the same conversation

117
docs/my-website/docs/completion/web_search.md
View file

@ -18,7 +18,7 @@ Each provider uses their own search backend:
| Provider | Search Engine | Notes |
|----------|---------------|-------|
| **OpenAI** (`gpt-4o-search-preview`, `gpt-4o-mini-search-preview`, `gpt-5-search-api`) | OpenAI's internal search | Real-time web data |
| **OpenAI** (`gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview`) | OpenAI's internal search | Real-time web data |
| **xAI** (`grok-3`) | xAI's search + X/Twitter | Real-time social media data |
| **Google AI/Vertex** (`gemini-2.0-flash`) | **Google Search** | Uses actual Google search results |
| **Anthropic** (`claude-3-5-sonnet`) | Anthropic's web search | Real-time web data |
@ -45,6 +45,19 @@ Use `web_search_options` when you need to:
**Anthropic Web Search Models**: Claude models that support web search: `claude-3-5-sonnet-latest`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-latest`, `claude-3-5-haiku-20241022`, `claude-3-7-sonnet-20250219`
:::
## OpenAI Web Search: Two Approaches
OpenAI offers two distinct ways to use web search depending on the endpoint and model:
| Approach | Endpoint | Models | How to enable |
|----------|----------|--------|---------------|
| **Search Models** | `/chat/completions` | `gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview` | Pass `web_search_options` parameter |
| **Web Search Tool** | `/responses` | `gpt-5`, `gpt-4.1`, `gpt-4o`, and other regular models | Pass `web_search_preview` tool |
:::tip Search models search automatically
Search models like `gpt-5-search-api` **automatically search the web** even without the `web_search_options` parameter. Use `web_search_options` to set `search_context_size` (`"low"`, `"medium"`, `"high"`) or specify `user_location` for localized results.
:::
## `/chat/completions` (litellm.completion)
### Quick Start
@ -56,7 +69,7 @@ Use `web_search_options` when you need to:
from litellm import completion
response = completion(
model="openai/gpt-4o-search-preview",
model="openai/gpt-5-search-api",
messages=[
{
"role": "user",
@ -76,31 +89,36 @@ response = completion(
```yaml
model_list:
# OpenAI
# OpenAI search models
- model_name: gpt-5-search-api
litellm_params:
model: openai/gpt-5-search-api
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-4o-search-preview
litellm_params:
model: openai/gpt-4o-search-preview
api_key: os.environ/OPENAI_API_KEY
# xAI
- model_name: grok-3
litellm_params:
model: xai/grok-3
api_key: os.environ/XAI_API_KEY
# Anthropic
- model_name: claude-3-5-sonnet-latest
litellm_params:
model: anthropic/claude-3-5-sonnet-latest
api_key: os.environ/ANTHROPIC_API_KEY
# VertexAI
- model_name: gemini-2-flash
litellm_params:
model: gemini-2.0-flash
vertex_project: your-project-id
vertex_location: us-central1
# Google AI Studio
- model_name: gemini-2-flash-studio
litellm_params:
@ -108,13 +126,13 @@ model_list:
api_key: os.environ/GOOGLE_API_KEY
```
2. Start the proxy
2. Start the proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
3. Test it!
```python showLineNumbers
from openai import OpenAI
@ -126,13 +144,18 @@ client = OpenAI(
)
response = client.chat.completions.create(
model="grok-3", # or any other web search enabled model
model="gpt-5-search-api", # or any other web search enabled model
messages=[
{
"role": "user",
"content": "What was a positive news story from today?"
}
]
],
extra_body={
"web_search_options": {
"search_context_size": "medium"
}
}
)
```
</TabItem>
@ -149,7 +172,7 @@ from litellm import completion
# Customize search context size
response = completion(
model="openai/gpt-4o-search-preview",
model="openai/gpt-5-search-api",
messages=[
{
"role": "user",
@ -257,6 +280,12 @@ response = client.chat.completions.create(
## `/responses` (litellm.responses)
Use the `web_search_preview` tool with models like `gpt-5`, `gpt-4.1`, `gpt-4o`, etc.
:::info
Search-dedicated models like `gpt-5-search-api` and `gpt-4o-search-preview` do **not** support the `/responses` endpoint. Use them with `/chat/completions` + `web_search_options` instead (see above).
:::
### Quick Start
<Tabs>
@ -266,18 +295,14 @@ response = client.chat.completions.create(
from litellm import responses
response = responses(
model="openai/gpt-4o",
input=[
{
"role": "user",
"content": "What was a positive news story from today?"
}
],
model="openai/gpt-5",
input="What is the capital of France?",
tools=[{
"type": "web_search_preview" # enables web search with default medium context size
}]
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
@ -285,19 +310,24 @@ response = responses(
```yaml
model_list:
- model_name: gpt-4o
- model_name: gpt-5
litellm_params:
model: openai/gpt-4o
model: openai/gpt-5
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-4.1
litellm_params:
model: openai/gpt-4.1
api_key: os.environ/OPENAI_API_KEY
```
2. Start the proxy
2. Start the proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
3. Test it!
```python showLineNumbers
from openai import OpenAI
@ -309,11 +339,11 @@ client = OpenAI(
)
response = client.responses.create(
model="gpt-4o",
model="gpt-5",
tools=[{
"type": "web_search_preview"
}],
input="What was a positive news story from today?",
input="What is the capital of France?",
)
print(response.output_text)
@ -331,13 +361,8 @@ from litellm import responses
# Customize search context size
response = responses(
model="openai/gpt-4o",
input=[
{
"role": "user",
"content": "What was a positive news story from today?"
}
],
model="openai/gpt-5",
input="What is the capital of France?",
tools=[{
"type": "web_search_preview",
"search_context_size": "low" # Options: "low", "medium" (default), "high"
@ -358,12 +383,12 @@ client = OpenAI(
# Customize search context size
response = client.responses.create(
model="gpt-4o",
model="gpt-5",
tools=[{
"type": "web_search_preview",
"search_context_size": "low" # Options: "low", "medium" (default), "high"
}],
input="What was a positive news story from today?",
input="What is the capital of France?",
)
print(response.output_text)
@ -417,14 +442,14 @@ model_list:
web_search_options:
search_context_size: "high" # Options: "low", "medium", "high"
# Different context size for different models
- model_name: gpt-4o-search-preview
# OpenAI search model with custom context size
- model_name: gpt-5-search-api
litellm_params:
model: openai/gpt-4o-search-preview
model: openai/gpt-5-search-api
api_key: os.environ/OPENAI_API_KEY
web_search_options:
search_context_size: "low"
# Gemini with medium context (default)
- model_name: gemini-2-flash
litellm_params:
@ -449,6 +474,7 @@ Use `litellm.supports_web_search(model="model_name")` -> returns `True` if model
```python showLineNumbers
# Check OpenAI models
assert litellm.supports_web_search(model="openai/gpt-5-search-api") == True
assert litellm.supports_web_search(model="openai/gpt-4o-search-preview") == True
# Check xAI models
@ -472,13 +498,20 @@ assert litellm.supports_web_search(model="gemini/gemini-2.0-flash") == True
```yaml
model_list:
# OpenAI
- model_name: gpt-5-search-api
litellm_params:
model: openai/gpt-5-search-api
api_key: os.environ/OPENAI_API_KEY
model_info:
supports_web_search: True
- model_name: gpt-4o-search-preview
litellm_params:
model: openai/gpt-4o-search-preview
api_key: os.environ/OPENAI_API_KEY
model_info:
supports_web_search: True
# xAI
- model_name: grok-3
litellm_params:
@ -533,6 +566,12 @@ Expected Response
```json showLineNumbers
{
"data": [
{
"model_group": "gpt-5-search-api",
"providers": ["openai"],
"max_tokens": 128000,
"supports_web_search": true
},
{
"model_group": "gpt-4o-search-preview",
"providers": ["openai"],

93
docs/my-website/docs/mcp_oauth.md
View file

@ -242,3 +242,96 @@ curl http://localhost:4000/mcp-rest/tools/call \
| `client_secret` | Yes | OAuth2 client secret. Supports `os.environ/VAR_NAME` |
| `token_url` | Yes | Token endpoint URL |
| `scopes` | No | List of scopes to request |
## Debugging OAuth
When the LiteLLM proxy is hosted remotely and you cannot access server logs, enable **debug headers** to get masked authentication diagnostics in the HTTP response.
### Enable Debug Mode
Add the `x-litellm-mcp-debug: true` header to your MCP client request.
**Claude Code:**
```bash
claude mcp add --transport http litellm_proxy http://proxy.example.com/atlassian_mcp/mcp \
--header "x-litellm-api-key: Bearer sk-..." \
--header "x-litellm-mcp-debug: true"
```
**curl:**
```bash
curl -X POST http://localhost:4000/atlassian_mcp/mcp \
-H "Content-Type: application/json" \
-H "x-litellm-api-key: Bearer sk-..." \
-H "x-litellm-mcp-debug: true" \
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
```
### Reading the Debug Response Headers
The response includes these headers (all sensitive values are masked):
| Header | Description |
|--------|-------------|
| `x-mcp-debug-inbound-auth` | Which inbound auth headers were present. |
| `x-mcp-debug-oauth2-token` | The OAuth2 token (masked). Shows `SAME_AS_LITELLM_KEY` if the LiteLLM key is leaking. |
| `x-mcp-debug-auth-resolution` | Which auth method was used: `oauth2-passthrough`, `m2m-client-credentials`, `per-request-header`, `static-token`, or `no-auth`. |
| `x-mcp-debug-outbound-url` | The upstream MCP server URL. |
| `x-mcp-debug-server-auth-type` | The `auth_type` configured on the server. |
**Example — healthy OAuth2 passthrough:**
```
x-mcp-debug-inbound-auth: x-litellm-api-key=Bearer****1234; authorization=Bearer****ef01
x-mcp-debug-oauth2-token: Bearer****ef01
x-mcp-debug-auth-resolution: oauth2-passthrough
x-mcp-debug-outbound-url: https://mcp.atlassian.com/v1/mcp
x-mcp-debug-server-auth-type: oauth2
```
**Example — LiteLLM key leaking (misconfigured):**
```
x-mcp-debug-inbound-auth: authorization=Bearer****1234
x-mcp-debug-oauth2-token: Bearer****1234 (SAME_AS_LITELLM_KEY - likely misconfigured)
x-mcp-debug-auth-resolution: oauth2-passthrough
x-mcp-debug-outbound-url: https://mcp.atlassian.com/v1/mcp
x-mcp-debug-server-auth-type: oauth2
```
### Common Issues
#### LiteLLM API key leaking to the MCP server
**Symptom:** `x-mcp-debug-oauth2-token` shows `SAME_AS_LITELLM_KEY`.
The `Authorization` header carries the LiteLLM API key instead of an OAuth2 token. The OAuth2 flow never ran because the client already had an `Authorization` header set.
**Fix:** Move the LiteLLM key to `x-litellm-api-key`:
```bash
# WRONG — blocks OAuth2 discovery
claude mcp add --transport http my_server http://proxy/mcp/server \
--header "Authorization: Bearer sk-..."
# CORRECT — LiteLLM key in dedicated header, Authorization free for OAuth2
claude mcp add --transport http my_server http://proxy/mcp/server \
--header "x-litellm-api-key: Bearer sk-..."
```
#### No OAuth2 token present
**Symptom:** `x-mcp-debug-oauth2-token` shows `(none)` and `x-mcp-debug-auth-resolution` shows `no-auth`.
Check that:
1. The `Authorization` header is NOT set as a static header in the client config.
2. The MCP server in LiteLLM config has `auth_type: oauth2`.
3. The `.well-known/oauth-protected-resource` endpoint returns valid metadata.
#### M2M token used instead of user token
**Symptom:** `x-mcp-debug-auth-resolution` shows `m2m-client-credentials`.
The server has `client_id`/`client_secret`/`token_url` configured so LiteLLM is fetching a machine-to-machine token instead of using the per-user OAuth2 token. To use per-user tokens, remove the client credentials from the server config.

47
docs/my-website/docs/mcp_troubleshoot.md
View file

@ -6,6 +6,39 @@ When LiteLLM acts as an MCP proxy, traffic normally flows `Client → LiteLLM Pr
For provisioning steps, transport options, and configuration fields, refer to [mcp.md](./mcp.md).
## Quick Start: Debug with One Command
The fastest way to debug MCP issues is to enable **debug headers**. Run this curl against your LiteLLM proxy and check the response headers:
```bash
curl -si -X POST http://localhost:4000/{your_mcp_server}/mcp \
-H "Content-Type: application/json" \
-H "x-litellm-api-key: Bearer sk-YOUR_KEY" \
-H "x-litellm-mcp-debug: true" \
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' \
2>&1 | grep -i "x-mcp-debug"
```
This returns masked diagnostic headers that tell you exactly what's happening with authentication:
```
x-mcp-debug-inbound-auth: x-litellm-api-key=Bearer****1234
x-mcp-debug-oauth2-token: Bearer****ef01
x-mcp-debug-auth-resolution: oauth2-passthrough
x-mcp-debug-outbound-url: https://mcp.atlassian.com/v1/mcp
x-mcp-debug-server-auth-type: oauth2
```
If you see `SAME_AS_LITELLM_KEY` in `x-mcp-debug-oauth2-token`, your LiteLLM API key is leaking to the MCP server instead of an OAuth2 token. See [Debugging OAuth](./mcp_oauth#debugging-oauth) for the fix and other common issues.
For Claude Code, add the debug header to your MCP config:
```bash
claude mcp add --transport http my_server http://localhost:4000/my_mcp/mcp \
--header "x-litellm-api-key: Bearer sk-..." \
--header "x-litellm-mcp-debug: true"
```
## Locate the Error Source
Pin down where the failure occurs before adjusting settings so you do not mix symptoms from separate hops.
@ -13,7 +46,7 @@ Pin down where the failure occurs before adjusting settings so you do not mix sy
### LiteLLM UI / Playground Errors (LiteLLM → MCP)
Failures shown on the MCP creation form or within the MCP Tool Testing Playground mean the LiteLLM proxy cannot reach the MCP server. Typical causes are misconfiguration (transport, headers, credentials), MCP/server outages, network/firewall blocks, or inaccessible OAuth metadata.
<Image
<Image
img={require('../img/mcp_tool_testing_playground.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
@ -22,7 +55,7 @@ Failures shown on the MCP creation form or within the MCP Tool Testing Playgroun
**Actions**
- Capture LiteLLM proxy logs alongside MCP-server logs (see [Error Log Example](./mcp_troubleshoot#error-log-example-failed-mcp-call)) to inspect the request/response pair and stack traces.
- From the LiteLLM server, run Method 2 ([`curl` smoke test](./mcp_troubleshoot#curl-smoke-test)) against the MCP endpoint to confirm basic connectivity.
- From the LiteLLM server, run a [`curl` smoke test](./mcp_troubleshoot#curl-smoke-test) against the MCP endpoint to confirm basic connectivity.
### Client Traffic Issues (Client → LiteLLM)
If only real client requests fail, determine whether LiteLLM ever reaches the MCP hop.
@ -43,7 +76,7 @@ During `/responses` or `/chat/completions`, LiteLLM may trigger MCP tool calls m
- Validate MCP connectivity with the [MCP Inspector](./mcp_troubleshoot#mcp-inspector) to ensure the server responds.
- Reproduce the same MCP call via the LiteLLM Playground to confirm LiteLLM can complete the MCP hop independently.
<Image
<Image
img={require('../img/mcp_playground.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
@ -55,6 +88,10 @@ LiteLLM performs metadata discovery per the MCP spec ([section 2.3](https://mode
- Use `curl <metadata_url>` (or similar) from the LiteLLM host to ensure the discovery document is reachable and contains the expected authorization/token endpoints.
- Record the exact metadata URL, requested scopes, and any static client credentials so support can replay the discovery step if needed.
## Debugging OAuth
For detailed OAuth2 debugging — including debug header reference, common misconfigurations, and example output — see [Debugging OAuth](./mcp_oauth#debugging-oauth).
## Verify Connectivity
Run lightweight validations before impacting production traffic.
@ -66,7 +103,7 @@ Use the MCP Inspector when you need to test both `Client → LiteLLM` and `Clien
2. Configure and connect:
- **Transport Type:** choose the transport the client uses (Streamable HTTP for LiteLLM).
- **URL:** the endpoint under test (LiteLLM MCP URL for `Client → LiteLLM`, or the MCP server URL for `Client → MCP`).
- **Custom Headers:** e.g., `Authorization: Bearer <LiteLLM API Key>`.
- **Custom Headers:** e.g., `x-litellm-api-key: Bearer <LiteLLM API Key>`.
3. Open the **Tools** tab and click **List Tools** to verify the MCP alias responds.
### `curl` Smoke Test
@ -79,7 +116,7 @@ curl -X POST https://your-target-domain.example.com/mcp \
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
```
Add `-H "Authorization: Bearer <LiteLLM API Key>"` when the target is a LiteLLM endpoint that requires authentication. Adjust the headers, or payload to target other MCP methods. Matching failures between `curl` and LiteLLM confirm that the MCP server or network/OAuth layer is the culprit.
Add `-H "x-litellm-api-key: Bearer <LiteLLM API Key>"` when the target is a LiteLLM endpoint that requires authentication. Adjust the headers or payload to target other MCP methods. Matching failures between `curl` and LiteLLM confirm that the MCP server or network/OAuth layer is the culprit.
## Review Logs

71
docs/my-website/docs/providers/anthropic.md
View file

@ -1473,6 +1473,20 @@ LiteLLM translates OpenAI's `reasoning_effort` to Anthropic's `thinking` paramet
| "medium" | "budget_tokens": 2048 |
| "high" | "budget_tokens": 4096 |
:::note
For Claude Opus 4.6, all `reasoning_effort` values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets, pass the native `thinking` parameter directly:
```python
from litellm import completion
resp = completion(
model="anthropic/claude-opus-4-6",
messages=[{"role": "user", "content": "What is the capital of France?"}],
thinking={"type": "enabled", "budget_tokens": 1024},
)
```
:::
<Tabs>
<TabItem value="sdk" label="SDK">
@ -1614,8 +1628,65 @@ curl http://0.0.0.0:4000/v1/chat/completions \
</TabItem>
</Tabs>
#### Adaptive Thinking (Claude Opus 4.6)
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.completion(
model="anthropic/claude-opus-4-6",
messages=[{"role": "user", "content": "What is the optimal strategy for solving this problem?"}],
thinking={"type": "adaptive"},
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "anthropic/claude-opus-4-6",
"messages": [{"role": "user", "content": "What is the optimal strategy for solving this problem?"}],
"thinking": {"type": "adaptive"}
}'
```
</TabItem>
</Tabs>
#### Enabled Thinking with Budget
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.completion(
model="anthropic/claude-opus-4-6",
messages=[{"role": "user", "content": "What is the capital of France?"}],
thinking={"type": "enabled", "budget_tokens": 5000},
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "anthropic/claude-opus-4-6",
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"thinking": {"type": "enabled", "budget_tokens": 5000}
}'
```
</TabItem>
</Tabs>
## **Passing Extra Headers to Anthropic API**

28
docs/my-website/docs/providers/dashscope.md
View file

@ -1,7 +1,7 @@
# Dashscope (Qwen API)
# Dashscope API (Qwen models)
https://dashscope.console.aliyun.com/
**We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests**
**We support ALL Qwen models (from Alibaba Cloud), just set `dashscope/` as a prefix when sending completion requests**
## API Key
```python
@ -9,6 +9,26 @@ https://dashscope.console.aliyun.com/
os.environ['DASHSCOPE_API_KEY']
```
## API Base
You can optionally specify the API base URL depending on your region:
| Region | API Base |
|--------|----------|
| **International** | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` |
| **China/Beijing** | `https://dashscope.aliyuncs.com/compatible-mode/v1` |
```python
# Set via environment variable
os.environ['DASHSCOPE_API_BASE'] = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
# Or pass directly in the completion call
response = completion(
model="dashscope/qwen-turbo",
messages=[{"role": "user", "content": "hello"}],
api_base="https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
)
```
## Sample Usage
```python
from litellm import completion
@ -43,9 +63,7 @@ for chunk in response:
```
## Supported Models - ALL Qwen Models Supported!
We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests
## All supported Models
[DashScope Model List](https://help.aliyun.com/zh/model-studio/compatibility-of-openai-with-dashscope?spm=a2c4g.11186623.help-menu-2400256.d_2_8_0.1efd516e2tTXBn&scm=20140722.H_2833609._.OR_help-T_cn~zh-V_1#7f9c78ae99pwz)

65
docs/my-website/docs/providers/openai.md
View file

@ -230,7 +230,70 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL
These also support the `OPENAI_BASE_URL` environment variable, which can be used to specify a custom API endpoint.
## OpenAI Vision Models
### OpenAI Web Search Models
OpenAI has two ways to use web search, depending on the endpoint:
| Approach | Endpoint | Models | How to enable |
|----------|----------|--------|---------------|
| **Search Models** | `/chat/completions` | `gpt-5-search-api`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview` | Pass `web_search_options` parameter |
| **Web Search Tool** | `/responses` | `gpt-5`, `gpt-4.1`, `gpt-4o`, and other regular models | Pass `web_search_preview` tool |
<Tabs>
<TabItem value="sdk-completion" label="SDK - /chat/completions">
```python showLineNumbers
from litellm import completion
response = completion(
model="openai/gpt-5-search-api",
messages=[{"role": "user", "content": "What is the capital of France?"}],
web_search_options={
"search_context_size": "medium" # Options: "low", "medium", "high"
}
)
```
</TabItem>
<TabItem value="sdk-responses" label="SDK - /responses">
```python showLineNumbers
from litellm import responses
response = responses(
model="openai/gpt-5",
input="What is the capital of France?",
tools=[{
"type": "web_search_preview",
"search_context_size": "low"
}]
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
# Search model for /chat/completions
- model_name: gpt-5-search-api
litellm_params:
model: openai/gpt-5-search-api
api_key: os.environ/OPENAI_API_KEY
# Regular model for /responses with web_search_preview tool
- model_name: gpt-5
litellm_params:
model: openai/gpt-5
api_key: os.environ/OPENAI_API_KEY
```
</TabItem>
</Tabs>
For full details, see the [Web Search guide](../completion/web_search.md).
## OpenAI Vision Models
| Model Name | Function Call |
|-----------------------|-----------------------------------------------------------------|
| gpt-4o | `response = completion(model="gpt-4o", messages=messages)` |

18
docs/my-website/docs/providers/openai/responses_api.md
View file

@ -37,6 +37,24 @@ for event in response:
print(event)
```
#### Web Search
```python showLineNumbers title="OpenAI Responses with Web Search"
import litellm
response = litellm.responses(
model="openai/gpt-5",
input="What is the capital of France?",
tools=[{
"type": "web_search_preview",
"search_context_size": "medium" # Options: "low", "medium", "high"
}]
)
print(response)
```
For full details, see the [Web Search guide](../../completion/web_search.md).
#### Image Generation with Streaming
```python showLineNumbers title="OpenAI Streaming Image Generation"
import litellm

62
docs/my-website/docs/providers/scaleway.md Normal file
View file

@ -0,0 +1,62 @@
# Scaleway
LiteLLM supports all [models available on Scaleway Generative APIs ↗](https://www.scaleway.com/en/docs/generative-apis/reference-content/supported-models/).
## Usage with LiteLLM Python SDK
```python
import os
from litellm import completion
os.environ["SCW_SECRET_KEY"] = "your-scaleway-secret-key"
messages = [{"role": "user", "content": "Write a short poem"}]
response = completion(model="scaleway/qwen3-235b-a22b-instruct-2507", messages=messages)
print(response)
```
## Usage with LiteLLM Proxy
### 1. Set Scaleway models in config.yaml
```yaml
model_list:
- model_name: scaleway-model
litellm_params:
model: scaleway/qwen3-235b-a22b-instruct-2507
api_key: "os.environ/SCW_SECRET_KEY" # ensure you have `SCW_SECRET_KEY` in your .env
```
### 2. Start proxy
```bash
litellm --config config.yaml
```
### 3. Query proxy
Assuming the proxy is running on [http://localhost:4000](http://localhost:4000):
```bash
curl http://localhost:4000/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer YOUR_LITELLM_MASTER_KEY" \
-d '{
"model": "scaleway-model",
"messages": [
{
"role": "system",
"content": "You are a helpful assistant."
},
{
"role": "user",
"content": "Write a short poem"
}
]
}'
```
`-H "Authorization: Bearer YOUR_LITELLM_MASTER_KEY" ` is only required if you have set a LiteLLM master key
## Supported features
Scaleway provider supports all features in [Generative APIs reference documentation ↗](https://www.scaleway.com/en/developers/api/generative-apis/), such as streaming, structured outputs and tool calling.

122
docs/my-website/docs/proxy/access_groups.md Normal file
View file

@ -0,0 +1,122 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Access Groups
Access Groups simplify how you define and manage resource access across your organization. Instead of configuring models, MCP servers, and agents separately on each key or team, you create one group that bundles the resources you want to grant, then attach that group to your keys or teams.
## Overview
**Access Groups** let you define a reusable set of allowed resources—models, MCP servers, and agents—in a single place. One group can grant access to all three resource types. Simply attach the group to a key or team, and they get access to everything defined in that group.
- **Unified resource control** – One group controls access to models, MCP servers, and agents together
- **Reusable** – Define once, attach to many keys or teams
- **Easy to maintain** – Update the group (add or remove resources) and all attached keys and teams automatically reflect the change
- **Clear visibility** – See exactly which resources each group grants and which keys/teams use it
<Image img={require('../../img/ui_access_groups.png')} />
### How It Works
**Key concept:** Define resources in a group → Attach group to key or team → Key/team gets access to all resources in the group
| Resource Type | What the group controls |
| --------------- | -------------------------------------------------------------------- |
| **Models** | Which LLM models keys/teams can use (e.g., `gpt-4`, `claude-3-opus`) |
| **MCP Servers** | Which MCP servers are available for tool calling |
| **Agents** | Which agents can be invoked |
## How to Create and Use Access Groups in the UI
### 1. Navigate to Access Groups
Go to the Admin UI (e.g. `http://localhost:4000/ui` or your `PROXY_BASE_URL/ui`) and click **Access Groups** in the sidebar.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/d117fdb2-18c8-49e0-91e6-1f830d2d4b85/ascreenshot_f5822a0ddac64e3383124419d0c66298_text_export.jpeg)
### 2. Create an Access Group
Click **Create Access Group** and give your group a name.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/aefb900d-d106-4436-806c-3608ad19659f/ascreenshot_3f6fed1256604fe3b7038a0778ce3342_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/0951bb93-61bd-477e-beaf-f58810f8980b/ascreenshot_f0fb5d552fd74ff8a1080e82758fcdc2_text_export.jpeg)
### 3. Define Resources in the Group
Use the tabs to select which models, MCP servers, and agents this group grants access to:
- **Models tab** – Select the LLM models
- **MCP Servers tab** – Select MCP servers (for tool calling)
- **Agents tab** – Select agents
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/37398e8f-cd50-48c9-85e2-c77b2eeb994b/ascreenshot_440ec7906c8f4199b30ef91c903960b9_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/99d36543-8582-4bb7-a34d-3d5fe0fcf12f/ascreenshot_d9983240955c496892e1f7c38c074045_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/06fc5919-5c71-4fc3-999b-da7a4800af3f/ascreenshot_db93fdf742b249dc90a4b9d5991d6097_text_export.jpeg)
### 4. Attach the Access Group to a Key
When creating or editing a virtual key, expand **Optional Settings** and select your Access Group. The key will inherit access to all models, MCP servers, and agents defined in that group.
1. Go to **Virtual Keys** and click **+ Create New Key**
2. Expand **Optional Settings**
3. In the Access Group field, select the group you created
4. Save the key
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/cdfa76ab-bf38-4ca4-a97d-2cb50fafe50b/ascreenshot_046daecb57554c28ba553cf6c01f5450_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/84f08e9c-e9d0-42aa-8317-f385190b6d7d/ascreenshot_2d239716d30f431d9ad494baf7933d6a_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/41d7b7f9-ac58-4602-b887-c35c9b419dce/ascreenshot_8abd4fef48014dd1b88848411e6d7912_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/e37b01c0-f2d7-4133-8b2f-ccc51f6769e1/ascreenshot_f495df428ad54cac9ec43b46c3dfc1b1_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-15/3fe33cad-6b64-46c3-a66e-6e6e073c3d7a/ascreenshot_f2dcc79ae8af47dd86ade2f85165d3c1_text_export.jpeg)
### 5. Attach the Access Group to a Team
You can also attach an Access Group to a team when creating or editing the team. All keys associated with that team will then have access to the resources defined in the group.
## Use Cases
### Team-based Access
Create groups like "Engineering", "Data Science", or "Product" with the models, MCP servers, and agents each team needs. Attach the group to the team—no need to configure each resource on every key.
### Environment Separation
- **Production group** – Production models, approved MCP servers, and production agents
- **Development group** – Cost-efficient models, experimental MCP tools, and dev agents
Attach the appropriate group to keys or teams based on environment.
### Simplified Onboarding
New developers get a key with an Access Group instead of manually configuring models, MCP servers, and agents. Add them to the right team or give them a key with the correct group.
### Centralized Updates
When you add a new model or MCP server to a group, every key and team attached to that group automatically gains access. Remove a resource from the group and it’s revoked everywhere at once.
## Access Group vs. Model Access Groups
LiteLLM has two related concepts:
| Feature | **Access Groups** (this page) | **Model Access Groups** |
| ---------- | ----------------------------------------------------------------------- | ------------------------------------------------------- |
| Definition | Define in the UI; one group can include models, MCP servers, and agents | Defined in config or via API; groups are model-centric |
| Scope | Models + MCP servers + agents | Models only |
| Attach to | Keys, teams | Keys, teams |
| Use when | You want unified control over models, MCP, and agents from the UI | You need config-based or API-based model access control |
For config-based model access with `access_groups` in `model_info`, see [Model Access Groups](./model_access_groups.md).
## Related Documentation
- [Virtual Keys](./virtual_keys.md) – Creating and managing API keys
- [Role-based Access Controls](./access_control.md) – Organizations, teams, and user roles
- [Model Access Groups](./model_access_groups.md) – Config-based model access groups
- [MCP Control](../mcp_control.md) – MCP server setup and access control

14
docs/my-website/docs/proxy/config_settings.md
View file

@ -520,6 +520,7 @@ router_settings:
| DEBUG_OTEL | Enable debug mode for OpenTelemetry
| DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3
| DEFAULT_A2A_AGENT_TIMEOUT | Default timeout in seconds for A2A (Agent-to-Agent) protocol requests. Default is 6000
| DEFAULT_ACCESS_GROUP_CACHE_TTL | Time-to-live in seconds for cached access group information. Default is 600 (10 minutes)
| DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096
| DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512
| DEFAULT_CHUNK_OVERLAP | Default chunk overlap for RAG text splitters. Default is 200
@ -548,6 +549,7 @@ router_settings:
| DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL | Default embedding model for MCP semantic tool filtering. Default is "text-embedding-3-small"
| DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD | Default similarity threshold for MCP semantic tool filtering. Default is 0.3
| DEFAULT_MCP_SEMANTIC_FILTER_TOP_K | Default number of top results to return for MCP semantic tool filtering. Default is 10
| MCP_NPM_CACHE_DIR | Directory for npm cache used by STDIO MCP servers. In containers the default (~/.npm) may not exist or be read-only. Default is `/tmp/.npm_mcp_cache`
| MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600
| MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200
| MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10
@ -555,7 +557,7 @@ router_settings:
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available**
| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy when `NUM_WORKERS` is not set. Default is 1. **We strongly recommend setting NUM_WORKERS to the number of vCPUs available** (e.g. `NUM_WORKERS=8` or `--num_workers 8`).
| DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7
| DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03
| DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0
@ -745,9 +747,12 @@ router_settings:
| LITERAL_API_KEY | API key for Literal integration
| LITERAL_API_URL | API URL for Literal service
| LITERAL_BATCH_SIZE | Batch size for Literal operations
| LITELLM_ANTHROPIC_BETA_HEADERS_URL | Custom URL for fetching Anthropic beta headers configuration. Default is the GitHub main branch URL
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
| LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker.
| LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
| LITELLM_DEPLOYMENT_ENVIRONMENT | Environment name for the deployment (e.g., "production", "staging"). Used as a fallback when OTEL_ENVIRONMENT_NAME is not set. Sets the `environment` tag in telemetry data
| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
| LITELLM_DD_LLM_OBS_PORT | Port for Datadog LLM Observability agent. Default is 8126
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
@ -760,11 +765,14 @@ router_settings:
| LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems.
| LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
| LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.
| LITELLM_UI_PATH | Path to directory for Admin UI files. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/ui` in Docker.
| LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval.
| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False`
| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
| LITELLM_LOCAL_POLICY_TEMPLATES | When set to "true", uses local backup policy templates instead of fetching from GitHub. Policy templates are fetched from https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json by default, with automatic fallback to local backup on failure
| LITELLM_LOG | Enable detailed logging for LiteLLM
| LITELLM_MODEL_COST_MAP_URL | URL for fetching model cost map data. Default is https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json
| LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file
@ -772,6 +780,10 @@ router_settings:
| LITELLM_METER_NAME | Name for OTEL Meter
| LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL
| LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL
| LITELLM_ENABLE_PYROSCOPE | If true, enables Pyroscope CPU profiling. Profiles are sent to PYROSCOPE_SERVER_ADDRESS. Off by default. See [Pyroscope profiling](/proxy/pyroscope_profiling).
| PYROSCOPE_APP_NAME | Application name reported to Pyroscope. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
| PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
| PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used.
| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers

1
docs/my-website/docs/proxy/configs.md
View file

@ -469,6 +469,7 @@ credential_list:
api_version: "2023-05-15"
credential_info:
description: "Production credentials for EU region"
custom_llm_provider: "azure"
```
#### Key Parameters

296
docs/my-website/docs/proxy/guardrails/policy_templates.md Normal file
View file

@ -0,0 +1,296 @@
# Policy Templates
Policy templates provide pre-configured guardrail policies that you can use as a starting point for your organization. Instead of manually creating policies and guardrails, you can select a template that matches your use case and deploy it with one click.
## Using Policy Templates
### In the UI
1. Navigate to **Policies → Templates** tab in the LiteLLM Admin UI
2. Browse available templates (e.g., "PII Protection", "Cost Control", "HR Compliance")
3. Click **"Use Template"** on any template
4. Review the guardrails that will be created:
- Existing guardrails are marked with a green checkmark
- New guardrails can be selected/deselected
5. Click **"Create X Guardrails & Use Template"**
6. Review and customize the pre-filled policy form
7. Click **"Create Policy"** to save
### Workflow
```
Select Template → Review Guardrails → Create Selected → Edit Policy → Save
```
The system automatically:
- ✅ Detects which guardrails already exist
- ✅ Creates only the missing guardrails you select
- ✅ Pre-fills the policy form with template data
- ✅ Lets you customize before saving
## Available Templates
Templates are fetched from [GitHub](https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json) with automatic fallback to local backup.
### Current Templates
#### 1. Advanced PII Protection (Australia)
- **Complexity:** High
- **Use Case:** Comprehensive PII detection for Australian organizations
- **Guardrails:**
- Australian tax identifiers (TFN, ABN, Medicare)
- Australian passports
- International PII (SSN, passports, national IDs)
- Contact information (email, phone, address)
- Financial data (credit cards, IBAN)
- API credentials (AWS, GitHub, Slack) - **BLOCKS** requests
- Network infrastructure (IP addresses)
- Protected class information (gender, race, religion, disability, etc.)
#### 2. Baseline PII Protection
- **Complexity:** Low
- **Use Case:** Basic protection for internal tools and testing
- **Guardrails:**
- Australian tax identifiers
- API credentials
- Financial data
## Creating Your Own Policy Templates
You can contribute policy templates for the entire LiteLLM community to use.
### Template Structure
Templates are defined in JSON format with the following structure:
```json
{
"id": "unique-template-id",
"title": "Display Title",
"description": "Detailed description of what this template protects",
"icon": "ShieldCheckIcon",
"iconColor": "text-purple-500",
"iconBg": "bg-purple-50",
"guardrails": [
"guardrail-name-1",
"guardrail-name-2"
],
"complexity": "Low|Medium|High",
"guardrailDefinitions": [
{
"guardrail_name": "example-guardrail",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{
"pattern_type": "prebuilt",
"pattern_name": "email",
"action": "MASK"
}
],
"pattern_redaction_format": "[{pattern_name}_REDACTED]"
},
"guardrail_info": {
"description": "What this guardrail does"
}
}
],
"templateData": {
"policy_name": "policy-name",
"description": "Policy description",
"guardrails_add": ["guardrail-name-1", "guardrail-name-2"],
"guardrails_remove": []
}
}
```
### Field Descriptions
#### Display Fields
- **id**: Unique identifier (lowercase with hyphens)
- **title**: User-facing name shown in UI
- **description**: Detailed explanation of what the template protects
- **icon**: Icon name (must be available in UI icon map)
- **iconColor**: Tailwind CSS text color class
- **iconBg**: Tailwind CSS background color class
- **guardrails**: Array of guardrail names (for display only)
- **complexity**: Badge showing difficulty ("Low", "Medium", or "High")
#### Guardrail Definitions
- **guardrailDefinitions**: Array of complete guardrail configurations
- Each must be a valid guardrail object that can be sent to `/guardrails` POST endpoint
- If a guardrail already exists, it will be skipped
- Can be empty `[]` if template uses only existing guardrails
#### Policy Configuration
- **templateData**: Object that pre-fills the policy form
- **policy_name**: Suggested name (user can edit)
- **description**: Policy description
- **guardrails_add**: Array of guardrail names to include
- **guardrails_remove**: Array to remove (usually `[]` for templates)
- **inherit**: (Optional) Parent policy name for inheritance
### Example Template
Here's a complete example for a HIPAA compliance template:
```json
{
"id": "hipaa-compliance",
"title": "HIPAA Compliance Policy",
"description": "Healthcare compliance policy that masks PHI and enforces HIPAA regulations for healthcare applications.",
"icon": "ShieldCheckIcon",
"iconColor": "text-red-500",
"iconBg": "bg-red-50",
"guardrails": [
"phi-detector",
"medical-record-blocker",
"patient-id-masker"
],
"complexity": "High",
"guardrailDefinitions": [
{
"guardrail_name": "phi-detector",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{
"pattern_type": "prebuilt",
"pattern_name": "us_ssn",
"action": "MASK"
},
{
"pattern_type": "prebuilt",
"pattern_name": "email",
"action": "MASK"
},
{
"pattern_type": "prebuilt",
"pattern_name": "us_phone",
"action": "MASK"
}
],
"pattern_redaction_format": "[PHI_REDACTED]"
},
"guardrail_info": {
"description": "Detects and masks Protected Health Information (PHI)"
}
}
],
"templateData": {
"policy_name": "hipaa-compliance-policy",
"description": "HIPAA compliance policy for healthcare applications",
"guardrails_add": [
"phi-detector",
"medical-record-blocker",
"patient-id-masker"
],
"guardrails_remove": []
}
}
```
## Contributing Templates
To contribute a policy template for everyone to use:
### Step 1: Create Your Template JSON
1. Create a JSON file following the structure above
2. Test it locally by adding it to your local `policy_templates.json`
3. Verify all guardrails work correctly
4. Ensure descriptions are clear and helpful
### Step 2: Submit a Pull Request
1. Fork the [LiteLLM repository](https://github.com/BerriAI/litellm)
2. Add your template to `policy_templates.json` at the root
3. Add your template to `litellm/policy_templates_backup.json` (keep both in sync)
4. Create a pull request with:
- Clear description of what the template protects
- Use case examples
- Any relevant compliance frameworks (HIPAA, GDPR, SOC 2, etc.)
### Guidelines
**DO:**
- ✅ Use clear, descriptive names
- ✅ Include comprehensive descriptions
- ✅ Test all guardrails thoroughly
- ✅ Document pattern sources (e.g., "Based on NIST guidelines")
- ✅ Group related guardrails logically
- ✅ Consider different complexity levels
**DON'T:**
- ❌ Include credentials or secrets
- ❌ Use overly broad patterns that may have false positives
- ❌ Duplicate existing templates
- ❌ Use custom code without thorough testing
## Using Templates Offline
For air-gapped or offline deployments, set the environment variable:
```bash
export LITELLM_LOCAL_POLICY_TEMPLATES=true
```
This forces the system to use the local backup (`litellm/policy_templates_backup.json`) instead of fetching from GitHub.
## Template Sources
- **GitHub (default):** https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json
- **Local backup:** `litellm/policy_templates_backup.json`
Templates are automatically fetched from GitHub on each request, with fallback to local backup on any failure.
## Available Pattern Types
When creating guardrails for templates, you can use these prebuilt patterns:
### Identity Documents
- `passport_australia`, `passport_us`, `passport_uk`, `passport_germany`, etc.
- `us_ssn`, `us_ssn_no_dash`
- `au_tfn`, `au_abn`, `au_medicare`
- `nl_bsn_contextual`
- `br_cpf`, `br_rg`, `br_cnpj`
### Financial
- `visa`, `mastercard`, `amex`, `discover`, `credit_card`
- `iban`
### Contact Information
- `email`
- `us_phone`, `br_phone_landline`, `br_phone_mobile`
- `street_address`
- `br_cep` (Brazilian postal code)
### Credentials
- `aws_access_key`, `aws_secret_key`
- `github_token`
- `slack_token`
- `generic_api_key`
### Network
- `ipv4`, `ipv6`
### Protected Class
- `gender_sexual_orientation`
- `race_ethnicity_national_origin`
- `religion`
- `age_discrimination`
- `disability`
- `marital_family_status`
- `military_status`
- `public_assistance`
See the [full patterns list](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json) for all available patterns.
## Related Docs
- [Guardrail Policies](./guardrail_policies)
- [Policy Tags](./policy_tags)
- [Content Filter Patterns](../hooks/content_filter)
- [Custom Code Guardrails](../hooks/custom_code)

43
docs/my-website/docs/proxy/pyroscope_profiling.md Normal file
View file

@ -0,0 +1,43 @@
# Grafana Pyroscope CPU profiling
LiteLLM proxy can send continuous CPU profiles to [Grafana Pyroscope](https://grafana.com/docs/pyroscope/latest/) when enabled via environment variables. This is optional and off by default.
## Quick start
1. **Install the optional dependency** (required only when enabling Pyroscope):
```bash
pip install pyroscope-io
```
Or install the proxy extra:
```bash
pip install "litellm[proxy]"
```
2. **Set environment variables** before starting the proxy:
| Variable | Required | Description |
|----------|----------|-------------|
| `LITELLM_ENABLE_PYROSCOPE` | Yes (to enable) | Set to `true` to enable Pyroscope profiling. |
| `PYROSCOPE_APP_NAME` | Yes (when enabled) | Application name shown in the Pyroscope UI. |
| `PYROSCOPE_SERVER_ADDRESS` | Yes (when enabled) | Pyroscope server URL (e.g. `http://localhost:4040`). |
| `PYROSCOPE_SAMPLE_RATE` | No | Sample rate (integer). If unset, the pyroscope-io library default is used. |
3. **Start the proxy**; profiling will begin automatically when the proxy starts.
```bash
export LITELLM_ENABLE_PYROSCOPE=true
export PYROSCOPE_APP_NAME=litellm-proxy
export PYROSCOPE_SERVER_ADDRESS=http://localhost:4040
litellm --config config.yaml
```
4. **View profiles** in the Pyroscope (or Grafana) UI and select your `PYROSCOPE_APP_NAME`.
## Notes
- **Optional dependency**: `pyroscope-io` is an optional dependency. If it is not installed and `LITELLM_ENABLE_PYROSCOPE=true`, the proxy will log a warning and continue without profiling.
- **Platform support**: The `pyroscope-io` package uses a native extension and is not available on all platforms (e.g. Windows is excluded by the package).
- **Other settings**: See [Configuration settings](/proxy/config_settings) for all proxy environment variables.

128
docs/my-website/docs/proxy/sync_anthropic_beta_headers.md Normal file
View file

@ -0,0 +1,128 @@
# Auto Sync Anthropic Beta Headers
Automatically keep your Anthropic beta headers configuration up to date without restarting your service. **This allows you to support new Anthropic beta features across all providers without restarting your service.**
## Overview
When Anthropic releases new beta features (e.g., new tool capabilities, extended context windows), you typically need to restart your LiteLLM service to get the latest beta header mappings for different providers (Anthropic, Bedrock, Vertex AI, Azure AI).
With auto-sync, LiteLLM automatically pulls the latest configuration from GitHub's [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) without requiring a restart. This means:
- **Zero downtime** when new beta features are released
- **Always up-to-date** provider support mappings
- **Automatic updates** - set it once and forget it
## Quick Start
**Manual sync:**
```bash
curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN" \
-H "Content-Type: application/json"
```
**Automatic sync every 24 hours:**
```bash
curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN" \
-H "Content-Type: application/json"
```
## API Endpoints
| Endpoint | Method | Description |
|----------|--------|-------------|
| `/reload/anthropic_beta_headers` | POST | Manual sync |
| `/schedule/anthropic_beta_headers_reload?hours={hours}` | POST | Schedule periodic sync |
| `/schedule/anthropic_beta_headers_reload` | DELETE | Cancel scheduled sync |
| `/schedule/anthropic_beta_headers_reload/status` | GET | Check sync status |
**Authentication:** Requires admin role or master key
## Python Example
```python
import requests
def sync_anthropic_beta_headers(proxy_url, admin_token):
response = requests.post(
f"{proxy_url}/reload/anthropic_beta_headers",
headers={"Authorization": f"Bearer {admin_token}"}
)
return response.json()
# Usage
result = sync_anthropic_beta_headers("https://your-proxy-url", "your-admin-token")
print(result['message'])
```
## Configuration
**Custom beta headers config URL:**
```bash
export LITELLM_ANTHROPIC_BETA_HEADERS_URL="https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json"
```
**Use local beta headers config:**
```bash
export LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True
```
## Scheduling Automatic Reloads
Schedule automatic reloads to ensure your proxy always has the latest beta header mappings:
```bash
# Reload every 24 hours
curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
**Check reload status:**
```bash
curl -X GET "https://your-proxy-url/schedule/anthropic_beta_headers_reload/status" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
**Response:**
```json
{
"scheduled": true,
"interval_hours": 24,
"last_run": "2026-02-13T10:00:00",
"next_run": "2026-02-14T10:00:00"
}
```
**Cancel scheduled reload:**
```bash
curl -X DELETE "https://your-proxy-url/schedule/anthropic_beta_headers_reload" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
## Environment Variables
| Variable | Description | Default |
|----------|-------------|---------|
| `LITELLM_ANTHROPIC_BETA_HEADERS_URL` | URL to fetch beta headers config from | GitHub main branch |
| `LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS` | Set to `True` to use local config only | `False` |
## How It Works
1. **Initial Load:** On startup, LiteLLM loads the beta headers configuration from the remote URL (or local file if configured)
2. **Caching:** The configuration is cached in memory to avoid repeated fetches on every request
3. **Scheduled Reload:** If configured, the proxy checks every 10 seconds whether it's time to reload based on your schedule
4. **Manual Reload:** You can trigger an immediate reload via the API endpoint
5. **Multi-Pod Support:** In multi-pod deployments, the reload configuration is stored in the database so all pods stay in sync
## Benefits
- **No Restarts Required:** Add support for new Anthropic beta features without downtime
- **Provider Compatibility:** Automatically get updated mappings for Bedrock, Vertex AI, Azure AI, etc.
- **Performance:** Configuration is cached and only reloaded when needed
- **Reliability:** Falls back to local configuration if remote fetch fails
## Related
- [Model Cost Map Sync](./sync_models_github.md) - Auto-sync model pricing data
- [Anthropic Beta Headers](../completion/anthropic.md#beta-features) - Using Anthropic beta features

128
docs/my-website/docs/response_api.md
View file

@ -1023,6 +1023,134 @@ curl http://localhost:4000/v1/responses \
## Server-side compaction
For long-running conversations, you can enable **server-side compaction** so that when the rendered context size crosses a threshold, the server automatically runs compaction in-stream and emits a compaction item—no separate `POST /v1/responses/compact` call is required.
Supported on the OpenAI Responses API when using the `openai` or `azure` provider. Pass `context_management` with a compaction entry and `compact_threshold` (token count; minimum 1000). When the context crosses the threshold, the server compacts in-stream and continues. Chain turns with `previous_response_id` or by appending output items to your next input array. See [OpenAI Compaction guide](https://developers.openai.com/api/docs/guides/compaction) for details.
For explicit control over when compaction runs, use the standalone compact endpoint (`POST /v1/responses/compact`) instead.
### Python SDK
```python showLineNumbers title="Server-side compaction with LiteLLM Python SDK"
import litellm
# Non-streaming: enable compaction when context exceeds 200k tokens
response = litellm.responses(
model="openai/gpt-4o",
input="Your conversation input...",
context_management=[{"type": "compaction", "compact_threshold": 200000}],
max_output_tokens=1024,
)
print(response)
# Streaming: same context_management, compaction runs in-stream if threshold is crossed
stream = litellm.responses(
model="openai/gpt-4o",
input="Your conversation input...",
context_management=[{"type": "compaction", "compact_threshold": 200000}],
stream=True,
)
for event in stream:
print(event)
```
### LiteLLM Proxy (AI Gateway)
Use the OpenAI SDK with your proxy as `base_url`, or call the proxy with curl. The proxy forwards `context_management` to the provider.
**OpenAI Python SDK (proxy as base_url):**
```python showLineNumbers title="Server-side compaction via LiteLLM Proxy"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000", # LiteLLM Proxy (AI Gateway)
api_key="your-proxy-api-key",
)
response = client.responses.create(
model="openai/gpt-4o",
input="Your conversation input...",
context_management=[{"type": "compaction", "compact_threshold": 200000}],
max_output_tokens=1024,
)
print(response)
```
**curl (proxy):**
```bash title="Server-side compaction via curl to LiteLLM Proxy"
curl -X POST "http://localhost:4000/v1/responses" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-d '{
"model": "openai/gpt-4o",
"input": "Your conversation input...",
"context_management": [{"type": "compaction", "compact_threshold": 200000}],
"max_output_tokens": 1024
}'
```
## Shell tool
The **Shell tool** lets the model run commands in a hosted container or local runtime (OpenAI Responses API). You pass `tools=[{"type": "shell", "environment": {...}}]`; the `environment` object configures the runtime (e.g. `type: "container_auto"` for auto-provisioned containers). See [OpenAI Shell tool guide](https://developers.openai.com/api/docs/guides/tools-shell) for full options.
Supported when using the `openai` or `azure` provider with a model that supports the Shell tool.
### Python SDK
```python showLineNumbers title="Shell tool with LiteLLM Python SDK"
import litellm
response = litellm.responses(
model="openai/gpt-5.2",
input="List files in /mnt/data and run python --version.",
tools=[{"type": "shell", "environment": {"type": "container_auto"}}],
tool_choice="auto",
max_output_tokens=1024,
)
```
### LiteLLM Proxy (AI Gateway)
Use the OpenAI SDK with your proxy as `base_url`, or call the proxy with curl. The proxy forwards `tools` (including `type: "shell"`) to the provider.
**OpenAI Python SDK (proxy as base_url):**
```python showLineNumbers title="Shell tool via LiteLLM Proxy"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-proxy-api-key",
)
response = client.responses.create(
model="openai/gpt-5.2",
input="List files in /mnt/data.",
tools=[{"type": "shell", "environment": {"type": "container_auto"}}],
tool_choice="auto",
max_output_tokens=1024,
)
```
**curl:**
```bash title="Shell tool via curl to LiteLLM Proxy"
curl -X POST "http://localhost:4000/v1/responses" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-d '{
"model": "openai/gpt-5.2",
"input": "List files in /mnt/data.",
"tools": [{"type": "shell", "environment": {"type": "container_auto"}}],
"tool_choice": "auto",
"max_output_tokens": 1024
}'
```
## Session Management
LiteLLM Proxy supports session management for all supported models. This allows you to store and fetch conversation history (state) in LiteLLM Proxy.

79
docs/my-website/docs/troubleshoot.md
View file

@ -1,102 +1,48 @@
# Troubleshooting & Support
## Information to Provide When Seeking Help
# Issue Reporting
When reporting issues, please include as much of the following as possible. It's okay if you can't provide everything—especially in production scenarios where the trigger might be unknown. Sharing most of this information will help us assist you more effectively.
### 1. LiteLLM Configuration File
## 1. LiteLLM Configuration File
Your `config.yaml` file (redact sensitive info like API keys). Include number of workers if not in config.
### 2. Initialization Command
## 2. Initialization Command
The command used to start LiteLLM (e.g., `litellm --config config.yaml --num_workers 8 --detailed_debug`).
### 3. LiteLLM Version
## 3. LiteLLM Version
- Current version
- Version when the issue first appeared (if different)
- Current version
- Version when the issue first appeared (if different)
- If upgraded, the version changed from → to
### 4. Environment Variables
## 4. Environment Variables
Non-sensitive environment variables not in your config (e.g., `NUM_WORKERS`, `LITELLM_LOG`, `LITELLM_MODE`). Do not include passwords or API keys.
### 5. Server Specifications
## 5. Server Specifications
CPU cores, RAM, OS, number of instances/replicas, etc.
### 6. Database and Redis Usage
## 6. Database and Redis Usage
- **Database:** Using database? (`DATABASE_URL` set), database type and version
- **Redis:** Using Redis? Redis version, configuration type (Standalone/Cluster/Sentinel).
### 7. Endpoints
## 7. Endpoints
The endpoint(s) you're using that are experiencing issues (e.g., `/chat/completions`, `/embeddings`).
### 8. Request Example
## 8. Request Example
A realistic example of the request causing issues, including expected vs. actual response and any error messages.
### 9. Error Logs, Stack Traces, and Metrics
## 9. Error Logs, Stack Traces, and Metrics
Full error logs, stack traces, and any images from service metrics (CPU, memory, request rates, etc.) that might help diagnose the issue.
---
## UI Issues
If you're experiencing issues with the LiteLLM Admin UI, please include the following information in addition to the general details above.
### 1. Steps to Reproduce
A clear, step-by-step description of how to trigger the issue (e.g., "Navigate to Settings → Team, click 'Create Team', fill in fields, click submit → error appears").
### 2. LiteLLM Version
The current version of LiteLLM you're running. Check via `litellm --version` or the UI's settings page.
### 3. Architecture & Deployment Setup
Distributed environments are a known source of UI issues. Please describe:
- **Number of LiteLLM instances/replicas** and how they are deployed (e.g., Kubernetes, Docker Compose, ECS)
- **Load balancer** type and configuration (e.g., ALB, Nginx, Cloudflare Tunnel) — include whether sticky sessions are enabled
- **How the UI is accessed** — directly via LiteLLM, through a reverse proxy, or behind an ingress controller
- **Any CDN or caching layers** between the user and the LiteLLM server
### 4. Network Tab Requests
Open your browser's Developer Tools (F12 → Network tab), reproduce the issue, and share:
- The **failing request(s)** — URL, method, status code, and response body
- **Screenshots or HAR export** of the relevant network activity
- Any **CORS or mixed-content errors** shown in the Console tab
### 5. Environment Variables
Non-sensitive environment variables related to the UI and proxy setup, such as:
- `LITELLM_MASTER_KEY`
- `PROXY_BASE_URL` / `LITELLM_PROXY_BASE_URL`
- `UI_BASE_PATH`
- Any SSO-related variables (e.g., `GOOGLE_CLIENT_ID`, `MICROSOFT_TENANT`)
Do **not** include passwords, secrets, or API keys.
### 6. Browser & Access Details
- **Browser** and version (e.g., Chrome 120, Firefox 121)
- **Access URL** used to reach the UI (redact sensitive parts)
- Whether the issue occurs for **all users or specific roles** (Admin, Internal User, etc.)
### 7. Screenshots or Screen Recordings
A screenshot or short screen recording of the issue is extremely helpful. Include any visible error messages, toasts, or unexpected behavior.
---
## Support Channels
[Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
@ -109,4 +55,3 @@ Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238
Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
[![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw)

49
docs/my-website/docs/troubleshoot/ui_issues.md Normal file
View file

@ -0,0 +1,49 @@
# UI Troubleshooting
If you're experiencing issues with the LiteLLM Admin UI, please include the following information when reporting.
## 1. Steps to Reproduce
A clear, step-by-step description of how to trigger the issue (e.g., "Navigate to Settings → Team, click 'Create Team', fill in fields, click submit → error appears").
## 2. LiteLLM Version
The current version of LiteLLM you're running. Check via `litellm --version` or the UI's settings page.
## 3. Architecture & Deployment Setup
Distributed environments are a known source of UI issues. Please describe:
- **Number of LiteLLM instances/replicas** and how they are deployed (e.g., Kubernetes, Docker Compose, ECS)
- **Load balancer** type and configuration (e.g., ALB, Nginx, Cloudflare Tunnel) — include whether sticky sessions are enabled
- **How the UI is accessed** — directly via LiteLLM, through a reverse proxy, or behind an ingress controller
- **Any CDN or caching layers** between the user and the LiteLLM server
## 4. Network Tab Requests
Open your browser's Developer Tools (F12 → Network tab), reproduce the issue, and share:
- The **failing request(s)** — URL, method, status code, and response body
- **Screenshots or HAR export** of the relevant network activity
- Any **CORS or mixed-content errors** shown in the Console tab
## 5. Environment Variables
Non-sensitive environment variables related to the UI and proxy setup, such as:
- `LITELLM_MASTER_KEY`
- `PROXY_BASE_URL` / `LITELLM_PROXY_BASE_URL`
- `UI_BASE_PATH`
- Any SSO-related variables (e.g., `GOOGLE_CLIENT_ID`, `MICROSOFT_TENANT`)
Do **not** include passwords, secrets, or API keys.
## 6. Browser & Access Details
- **Browser** and version (e.g., Chrome 120, Firefox 121)
- **Access URL** used to reach the UI (redact sensitive parts)
- Whether the issue occurs for **all users or specific roles** (Admin, Internal User, etc.)
## 7. Screenshots or Screen Recordings
A screenshot or short screen recording of the issue is extremely helpful. Include any visible error messages, toasts, or unexpected behavior.

55
docs/my-website/docs/tutorials/claude_code_beta_headers.md
View file

@ -92,9 +92,34 @@ Open `anthropic_beta_headers_config.json` and add the new header to each provide
- **Header transformations**: Some providers use different header names (e.g., Bedrock maps `advanced-tool-use-2025-11-20` to `tool-search-tool-2025-10-19`)
- **Alphabetical order**: Keep headers sorted alphabetically for maintainability
### Step 3: Restart Your Application
### Step 3: Reload Configuration (No Restart Required!)
After updating the config file, restart your LiteLLM proxy or application:
**Option 1: Dynamic Reload Without Restart**
Instead of restarting your application, you can dynamically reload the beta headers configuration using environment variables and API endpoints:
```bash
# Set environment variable to fetch from remote URL (Do this if you want to point it to some other URL)
export LITELLM_ANTHROPIC_BETA_HEADERS_URL="https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json"
# Manually trigger reload via API (no restart needed!)
curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
**Option 2: Schedule Automatic Reloads**
Set up automatic reloading to always stay up-to-date with the latest beta headers:
```bash
# Reload configuration every 24 hours
curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
-H "Authorization: Bearer YOUR_ADMIN_TOKEN"
```
**Option 3: Traditional Restart**
If you prefer the traditional approach, restart your LiteLLM proxy or application:
```bash
# If using LiteLLM proxy
@ -104,7 +129,11 @@ litellm --config config.yaml
# Just restart your Python application
```
The updated configuration will be loaded automatically.
:::tip Zero-Downtime Updates
With dynamic reloading, you can fix invalid beta header errors **without restarting your service**! This is especially useful in production environments where downtime is costly.
See [Auto Sync Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) for complete documentation.
:::
## Fixing Invalid Beta Header Errors
@ -215,6 +244,26 @@ Result sent to Bedrock:
anthropic-beta: computer-use-2025-01-24
```
## Dynamic Configuration Management (No Restart Required!)
### Environment Variables
Control how LiteLLM loads the beta headers configuration:
| Variable | Description | Default |
|----------|-------------|---------|
| `LITELLM_ANTHROPIC_BETA_HEADERS_URL` | URL to fetch config from | GitHub main branch |
| `LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS` | Set to `True` to use local config only | `False` |
**Example: Use Custom Config URL**
```bash
export LITELLM_ANTHROPIC_BETA_HEADERS_URL="https://your-company.com/custom-beta-headers.json"
```
**Example: Use Local Config Only (No Remote Fetching)**
```bash
export LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True
```
## Provider-Specific Notes
### Bedrock

72
docs/my-website/docs/tutorials/claude_mcp.md
View file

@ -9,7 +9,7 @@ Note: LiteLLM supports OAuth for MCP servers as well. [Learn more](https://docs.
## Connecting MCP Servers
You can also connect MCP servers to Claude Code via LiteLLM Proxy.
You can connect MCP servers to Claude Code via LiteLLM Proxy.
1. Add the MCP server to your `config.yaml`
@ -23,6 +23,7 @@ In this example, we'll add the Github MCP server to our `config.yaml`
mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
transport: "http"
auth_type: oauth2
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
@ -34,31 +35,70 @@ mcp_servers:
In this example, we'll add the Atlassian MCP server to our `config.yaml`
```yaml title="config.yaml" showLineNumbers
atlassian_mcp:
server_id: atlassian_mcp_id
url: "https://mcp.atlassian.com/v1/sse"
transport: "sse"
auth_type: oauth2
mcp_servers:
atlassian_mcp:
url: "https://mcp.atlassian.com/v1/mcp"
transport: "http"
auth_type: oauth2
```
</TabItem>
</Tabs>
:::important
The server name under `mcp_servers:` (e.g. `atlassian_mcp`, `github_mcp`) **must match** the name used in the Claude Code URL path (`/mcp/<server_name>`). A mismatch will cause a 404 error during OAuth.
:::
2. Start LiteLLM Proxy
Since Claude Code needs a publicly accessible URL for the OAuth callback, expose your proxy via ngrok or a similar tool.
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Use the MCP server in Claude Code
```bash
claude mcp add --transport http litellm_proxy http://0.0.0.0:4000/github_mcp/mcp --header "Authorization: Bearer sk-LITELLM_VIRTUAL_KEY"
# In a separate terminal — expose proxy for OAuth callbacks
ngrok http 4000
```
For MCP servers that require dynamic client registration (such as Atlassian), please set `x-litellm-api-key: Bearer sk-LITELLM_VIRTUAL_KEY` instead of using `Authorization: Bearer LITELLM_VIRTUAL_KEY`.
3. Add the MCP server to Claude Code
<Tabs>
<TabItem value="github" label="GitHub MCP">
```bash
claude mcp add --transport http litellm-github https://your-ngrok-url.ngrok-free.dev/mcp/github_mcp \
--header "x-litellm-api-key: Bearer sk-1234"
```
</TabItem>
<TabItem value="atlassian" label="Atlassian MCP">
```bash
claude mcp add --transport http litellm-atlassian https://your-ngrok-url.ngrok-free.dev/mcp/atlassian_mcp \
--header "x-litellm-api-key: Bearer sk-1234"
```
</TabItem>
</Tabs>
**Parameter breakdown:**
| Parameter | Description |
|-----------|-------------|
| `--transport http` | Use HTTP transport for the MCP connection |
| `litellm-atlassian` | The name for this MCP server **on Claude Code** — can be anything you choose |
| `https://your-ngrok-url.ngrok-free.dev/mcp/atlassian_mcp` | The LiteLLM proxy URL. Format: `<PROXY_URL>/mcp/<server_name_on_litellm>`. The `atlassian_mcp` part **must match** the key under `mcp_servers:` in your LiteLLM proxy config |
| `--header "x-litellm-api-key: Bearer sk-1234"` | Your LiteLLM virtual key for authentication to the proxy |
You can also add the MCP server directly to your `~/.claude.json` file instead of using `claude mcp add`. [See Claude Code docs](https://docs.anthropic.com/en/docs/claude-code/mcp).
:::note
For MCP servers that require OAuth (such as Atlassian), use `x-litellm-api-key` instead of `Authorization` for the LiteLLM virtual key. The `Authorization` header is reserved for the OAuth flow.
:::
4. Authenticate via Claude Code
@ -68,24 +108,20 @@ a. Start Claude Code
claude
```
b. Authenticate via Claude Code
b. Open the MCP menu
```bash
/mcp
```
c. Select the MCP server
c. Select the MCP server (e.g. `litellm-atlassian`)
```bash
> litellm_proxy
```
d. Start Oauth flow via Claude Code
d. Start the OAuth flow
```bash
> 1. Authenticate
2. Reconnect
3. Disable
3. Disable
```
e. Once completed, you should see this success message:

BIN
docs/my-website/img/release_notes/guard_actions.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 506 KiB

BIN
docs/my-website/img/ui_access_groups.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 338 KiB

7
docs/my-website/package-lock.json generated
View file

@ -20455,6 +20455,13 @@
"url": "https://opencollective.com/webpack"
}
},
"node_modules/search-insights": {
"version": "2.17.3",
"resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz",
"integrity": "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==",
"license": "MIT",
"peer": true
},
"node_modules/section-matter": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz",

426
docs/my-website/release_notes/v1.81.12.md Normal file
View file

@ -0,0 +1,426 @@
---
title: "[Preview] v1.81.12 - Guardrail Policy Templates & Action Builder"
slug: "v1-81-12"
date: 2026-02-14T00:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
## Deploy this version
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.81.12.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.81.12.rc1
```
</TabItem>
</Tabs>
## Key Highlights
- **Policy Templates** - [Pre-configured guardrail policy templates for common safety and compliance use-cases (including NSFW, toxic content, and child safety)](../../docs/proxy/guardrails/policy_templates)
- **Guardrail Action Builder** - [Build and customize guardrail policy flows with the new action-builder UI and conditional execution support](../../docs/proxy/guardrails/policy_templates)
- **MCP OAuth2 M2M + Tracing** - [Add machine-to-machine OAuth2 support for MCP servers and OpenTelemetry tracing for MCP calls through AI Gateway](../../docs/mcp)
- **Responses API `shell` Tool & `context_management` support** - [Server-side context management (compaction) and Shell tool support for the OpenAI Responses API](../../docs/response_api)
- **Access Groups** - [Create access groups to manage model, MCP server, and agent access across teams and keys](../../docs/proxy/access_groups)
- **50+ New Bedrock Regional Model Entries** - DeepSeek V3.2, MiniMax M2.1, Kimi K2.5, Qwen3 Coder Next, and NVIDIA Nemotron Nano across multiple regions
---
## Guardrail Action Builder
This release adds a visual action builder for guardrail policies with conditional execution support. You can now chain guardrails into multi-step pipelines — if a simple guardrail fails, route to an advanced one instead of immediately blocking. Each step has configurable ON PASS and ON FAIL actions (Next Step, Block, or Allow), and you can test the full pipeline with a sample message before saving.
![Guardrail Action Builder](../img/release_notes/guard_actions.png)
### Access Groups
Access Groups simplify defining resource access across your organization. One group can grant access to models, MCP servers, and agents—simply attach it to a key or team. Create groups in the Admin UI, define which resources each group includes, then assign the group when creating keys or teams. Updates to a group apply automatically to all attached keys and teams.
<Image img={require('../img/ui_access_groups.png')} />
## New Providers and Endpoints
### New Providers (2 new providers)
| Provider | Supported LiteLLM Endpoints | Description |
| -------- | --------------------------- | ----------- |
| [Scaleway](../../docs/providers/scaleway) | `/chat/completions` | Scaleway Generative APIs for chat completions |
| [Sarvam AI](../../docs/providers/sarvam) | `/chat/completions`, `/audio/transcriptions`, `/audio/speech` | Sarvam AI STT and TTS support for Indian languages |
---
## New Models / Updated Models
#### New Model Support (19 highlighted models)
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) |
| -------- | ----- | -------------- | ------------------- | -------------------- |
| AWS Bedrock | `deepseek.v3.2` | 164K | $0.62 | $1.85 |
| AWS Bedrock | `minimax.minimax-m2.1` | 196K | $0.30 | $1.20 |
| AWS Bedrock | `moonshotai.kimi-k2.5` | 262K | $0.60 | $3.00 |
| AWS Bedrock | `moonshotai.kimi-k2-thinking` | 262K | $0.73 | $3.03 |
| AWS Bedrock | `qwen.qwen3-coder-next` | 262K | $0.50 | $1.20 |
| AWS Bedrock | `nvidia.nemotron-nano-3-30b` | 262K | $0.06 | $0.24 |
| Azure AI | `azure_ai/kimi-k2.5` | 262K | $0.60 | $3.00 |
| Vertex AI | `vertex_ai/zai-org/glm-5-maas` | 200K | $1.00 | $3.20 |
| MiniMax | `minimax/MiniMax-M2.5` | 1M | $0.30 | $1.20 |
| MiniMax | `minimax/MiniMax-M2.5-lightning` | 1M | $0.30 | $2.40 |
| Dashscope | `dashscope/qwen3-max` | 258K | Tiered pricing | Tiered pricing |
| Perplexity | `perplexity/preset/pro-search` | - | Per-request | Per-request |
| Perplexity | `perplexity/openai/gpt-4o` | - | Per-request | Per-request |
| Perplexity | `perplexity/openai/gpt-5.2` | - | Per-request | Per-request |
| Vercel AI Gateway | `vercel_ai_gateway/anthropic/claude-opus-4.6` | 200K | $5.00 | $25.00 |
| Vercel AI Gateway | `vercel_ai_gateway/anthropic/claude-sonnet-4` | 200K | $3.00 | $15.00 |
| Vercel AI Gateway | `vercel_ai_gateway/anthropic/claude-haiku-4.5` | 200K | $1.00 | $5.00 |
| Sarvam AI | `sarvam/sarvam-m` | 8K | Free tier | Free tier |
| Anthropic | `fast/claude-opus-4-6` | 1M | $30.00 | $150.00 |
*Note: AWS Bedrock models are available across multiple regions (us-east-1, us-east-2, us-west-2, eu-central-1, eu-north-1, ap-northeast-1, ap-south-1, ap-southeast-3, sa-east-1). 54 regional model entries were added in total.*
#### Features
- **[Anthropic](../../docs/providers/anthropic)**
- Enable non-tool structured outputs on Claude Opus 4.5 and 4.6 using `output_format` param - [PR #20548](https://github.com/BerriAI/litellm/pull/20548)
- Add support for `anthropic_messages` call type in prompt caching - [PR #19233](https://github.com/BerriAI/litellm/pull/19233)
- Managing Anthropic Beta Headers with remote URL fetching - [PR #20935](https://github.com/BerriAI/litellm/pull/20935), [PR #21110](https://github.com/BerriAI/litellm/pull/21110)
- Remove `x-anthropic-billing` block - [PR #20951](https://github.com/BerriAI/litellm/pull/20951)
- Use Authorization Bearer for OAuth tokens instead of x-api-key - [PR #21039](https://github.com/BerriAI/litellm/pull/21039)
- Filter unsupported JSON schema constraints for structured outputs - [PR #20813](https://github.com/BerriAI/litellm/pull/20813)
- New Claude Opus 4.6 features for `/v1/messages` - [PR #20733](https://github.com/BerriAI/litellm/pull/20733)
- Fix `reasoning_effort=None` and `"none"` should return None for Opus 4.6 - [PR #20800](https://github.com/BerriAI/litellm/pull/20800)
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Extend model support with 4 new beta models - [PR #21035](https://github.com/BerriAI/litellm/pull/21035)
- Add Claude Opus 4.6 to `_supports_tool_search_on_bedrock` - [PR #21017](https://github.com/BerriAI/litellm/pull/21017)
- Correct Bedrock Claude Opus 4.6 model IDs (remove `:0` suffix) - [PR #20564](https://github.com/BerriAI/litellm/pull/20564), [PR #20671](https://github.com/BerriAI/litellm/pull/20671)
- Add `output_config` as supported param - [PR #20748](https://github.com/BerriAI/litellm/pull/20748)
- **[Vertex AI](../../docs/providers/vertex)**
- Add Vertex GLM-5 model support - [PR #21053](https://github.com/BerriAI/litellm/pull/21053)
- Propagate `extra_headers` anthropic-beta to request body - [PR #20666](https://github.com/BerriAI/litellm/pull/20666)
- Preserve `usageMetadata` in `_hidden_params` - [PR #20559](https://github.com/BerriAI/litellm/pull/20559)
- Map `IMAGE_PROHIBITED_CONTENT` to `content_filter` - [PR #20524](https://github.com/BerriAI/litellm/pull/20524)
- Add RAG ingest for Vertex AI - [PR #21120](https://github.com/BerriAI/litellm/pull/21120)
- **[OCI / Cohere](../../docs/providers/cohere)**
- OCI Cohere responseFormat/Pydantic support - [PR #20663](https://github.com/BerriAI/litellm/pull/20663)
- Fix OCI Cohere system messages by populating `preambleOverride` - [PR #20958](https://github.com/BerriAI/litellm/pull/20958)
- **[Perplexity](../../docs/providers/perplexity)**
- Perplexity Research API support with preset search - [PR #20860](https://github.com/BerriAI/litellm/pull/20860)
- **[MiniMax](../../docs/providers/minimax)**
- Add MiniMax-M2.5 and MiniMax-M2.5-lightning models - [PR #21054](https://github.com/BerriAI/litellm/pull/21054)
- **[Kimi / Moonshot](../../docs/providers/moonshot)**
- Add Kimi model pricing by region - [PR #20855](https://github.com/BerriAI/litellm/pull/20855)
- Add `moonshotai.kimi-k2.5` - [PR #20863](https://github.com/BerriAI/litellm/pull/20863)
- **[Dashscope](../../docs/providers/dashscope)**
- Add `dashscope/qwen3-max` model with tiered pricing - [PR #20919](https://github.com/BerriAI/litellm/pull/20919)
- **[Vercel AI Gateway](../../docs/providers/vercel_ai_gateway)**
- Add new Vercel AI Anthropic models - [PR #20745](https://github.com/BerriAI/litellm/pull/20745)
- **[Azure AI](../../docs/providers/azure_ai)**
- Add `azure_ai/kimi-k2.5` to Azure model DB - [PR #20896](https://github.com/BerriAI/litellm/pull/20896)
- Support Azure AD token auth for non-Claude azure_ai models - [PR #20981](https://github.com/BerriAI/litellm/pull/20981)
- Fix Azure batches issues - [PR #21092](https://github.com/BerriAI/litellm/pull/21092)
- **[DeepSeek](../../docs/providers/deepseek)**
- Sync DeepSeek model metadata and add bare-name fallback - [PR #20938](https://github.com/BerriAI/litellm/pull/20938)
- **[Gemini](../../docs/providers/gemini)**
- Handle image in assistant message for Gemini - [PR #20845](https://github.com/BerriAI/litellm/pull/20845)
- Add missing tpm/rpm for Gemini models - [PR #21175](https://github.com/BerriAI/litellm/pull/21175)
- **General**
- Add 30 missing models to pricing JSON - [PR #20797](https://github.com/BerriAI/litellm/pull/20797)
- Cleanup 39 deprecated OpenRouter models - [PR #20786](https://github.com/BerriAI/litellm/pull/20786)
- Standardize endpoint `display_name` naming convention - [PR #20791](https://github.com/BerriAI/litellm/pull/20791)
- Fix and stabilize model cost map formatting - [PR #20895](https://github.com/BerriAI/litellm/pull/20895)
- Export `PermissionDeniedError` from `litellm.__init__` - [PR #20960](https://github.com/BerriAI/litellm/pull/20960)
### Bug Fixes
- **[Anthropic](../../docs/providers/anthropic)**
- Fix `get_supported_anthropic_messages_params` - [PR #20752](https://github.com/BerriAI/litellm/pull/20752)
- Fix `base_model` name for body and deployment name in URL - [PR #20747](https://github.com/BerriAI/litellm/pull/20747)
- **[Azure](../../docs/providers/azure/azure)**
- Preserve `content_policy_violation` error details from Azure OpenAI - [PR #20883](https://github.com/BerriAI/litellm/pull/20883)
- **[Vertex AI](../../docs/providers/vertex)**
- Fix Gemini multi-turn tool calling message formatting (added and reverted) - [PR #20569](https://github.com/BerriAI/litellm/pull/20569), [PR #21051](https://github.com/BerriAI/litellm/pull/21051)
---
## LLM API Endpoints
#### Features
- **[Responses API](../../docs/response_api)**
- Add server-side context management (compaction) support - [PR #21058](https://github.com/BerriAI/litellm/pull/21058)
- Add Shell tool support for OpenAI Responses API - [PR #21063](https://github.com/BerriAI/litellm/pull/21063)
- Preserve tool call argument deltas when streaming id is omitted - [PR #20712](https://github.com/BerriAI/litellm/pull/20712)
- Preserve interleaved thinking/redacted_thinking blocks during streaming - [PR #20702](https://github.com/BerriAI/litellm/pull/20702)
- **[Chat Completions](../../docs/completion/input)**
- Add Web Search support using LiteLLM `/search` (web search interception hook) - [PR #20483](https://github.com/BerriAI/litellm/pull/20483)
- Preserved nullable object fields by carrying schema properties - [PR #19132](https://github.com/BerriAI/litellm/pull/19132)
- Support `prompt_cache_key` for OpenAI and Azure chat completions - [PR #20989](https://github.com/BerriAI/litellm/pull/20989)
- **[Pass-Through Endpoints](../../docs/pass_through/bedrock)**
- Add support for `langchain_aws` via LiteLLM passthrough - [PR #20843](https://github.com/BerriAI/litellm/pull/20843)
- Add `custom_body` parameter to `endpoint_func` in `create_pass_through_route` - [PR #20849](https://github.com/BerriAI/litellm/pull/20849)
- **[Vector Stores](../../docs/providers/openai)**
- Add `target_model_names` for vector store endpoints - [PR #21089](https://github.com/BerriAI/litellm/pull/21089)
- **General**
- Add `output_config` as supported param - [PR #20748](https://github.com/BerriAI/litellm/pull/20748)
- Add managed error file support - [PR #20838](https://github.com/BerriAI/litellm/pull/20838)
#### Bugs
- **General**
- Stop leaking Python tracebacks in streaming SSE error responses - [PR #20850](https://github.com/BerriAI/litellm/pull/20850)
- Fix video list pagination cursors not encoded with provider metadata - [PR #20710](https://github.com/BerriAI/litellm/pull/20710)
- Handle `metadata=None` in SDK path retry/error logic - [PR #20873](https://github.com/BerriAI/litellm/pull/20873)
- Fix Spend logs pickle error with Pydantic models and redaction - [PR #20685](https://github.com/BerriAI/litellm/pull/20685)
- Remove duplicate `PerplexityResponsesConfig` from `LLM_CONFIG_NAMES` - [PR #21105](https://github.com/BerriAI/litellm/pull/21105)
---
## Management Endpoints / UI
#### Features
- **Access Groups**
- New Access Groups feature for managing model, MCP server, and agent access - [PR #21022](https://github.com/BerriAI/litellm/pull/21022)
- Access Groups table and details page UI - [PR #21165](https://github.com/BerriAI/litellm/pull/21165)
- Refactor `model_ids` to `model_names` for backwards compatibility - [PR #21166](https://github.com/BerriAI/litellm/pull/21166)
- **Policies**
- Allow connecting Policies to Tags, simulating Policies, viewing key/team counts - [PR #20904](https://github.com/BerriAI/litellm/pull/20904)
- Guardrail pipeline support for conditional sequential execution - [PR #21177](https://github.com/BerriAI/litellm/pull/21177)
- Pipeline flow builder UI for guardrail policies - [PR #21188](https://github.com/BerriAI/litellm/pull/21188)
- **SSO / Auth**
- New Login With SSO Button - [PR #20908](https://github.com/BerriAI/litellm/pull/20908)
- M2M OAuth2 UI Flow - [PR #20794](https://github.com/BerriAI/litellm/pull/20794)
- Allow Organization and Team Admins to call `/invitation/new` - [PR #20987](https://github.com/BerriAI/litellm/pull/20987)
- Invite User: Email Integration Alert - [PR #20790](https://github.com/BerriAI/litellm/pull/20790)
- Populate identity fields in proxy admin JWT early-return path - [PR #21169](https://github.com/BerriAI/litellm/pull/21169)
- **Spend Logs**
- Show predefined error codes in filter with user definable fallback - [PR #20773](https://github.com/BerriAI/litellm/pull/20773)
- Paginated searchable model select - [PR #20892](https://github.com/BerriAI/litellm/pull/20892)
- Sorting columns support - [PR #21143](https://github.com/BerriAI/litellm/pull/21143)
- Allow sorting on `/spend/logs/ui` - [PR #20991](https://github.com/BerriAI/litellm/pull/20991)
- **UI Improvements**
- Navbar: Option to hide Usage Popup - [PR #20910](https://github.com/BerriAI/litellm/pull/20910)
- Model Page: Improve Credentials Messaging - [PR #21076](https://github.com/BerriAI/litellm/pull/21076)
- Fallbacks: Default configurable to 10 models - [PR #21144](https://github.com/BerriAI/litellm/pull/21144)
- Fallback display with arrows and card structure - [PR #20922](https://github.com/BerriAI/litellm/pull/20922)
- Team Info: Migrate to AntD Tabs + Table - [PR #20785](https://github.com/BerriAI/litellm/pull/20785)
- AntD refactoring and 0 cost models fix - [PR #20687](https://github.com/BerriAI/litellm/pull/20687)
- Zscaler AI Guard UI - [PR #21077](https://github.com/BerriAI/litellm/pull/21077)
- Include Config Defined Pass Through Endpoints - [PR #20898](https://github.com/BerriAI/litellm/pull/20898)
- Rename "HTTP" to "Streamable HTTP (Recommended)" in MCP server page - [PR #21000](https://github.com/BerriAI/litellm/pull/21000)
- MCP server discovery UI - [PR #21079](https://github.com/BerriAI/litellm/pull/21079)
- **Virtual Keys**
- Allow Management keys to access `user/daily/activity` and team - [PR #20124](https://github.com/BerriAI/litellm/pull/20124)
- Skip premium check for empty metadata fields on team/key update - [PR #20598](https://github.com/BerriAI/litellm/pull/20598)
#### Bugs
- Logs: Fix Input and Output Copying - [PR #20657](https://github.com/BerriAI/litellm/pull/20657)
- Teams: Fix Available Teams - [PR #20682](https://github.com/BerriAI/litellm/pull/20682)
- Spend Logs: Reset Filters Resets Custom Date Range - [PR #21149](https://github.com/BerriAI/litellm/pull/21149)
- Usage: Request Chart stack variant fix - [PR #20894](https://github.com/BerriAI/litellm/pull/20894)
- Add Auto Router: Description Text Input Focus - [PR #21004](https://github.com/BerriAI/litellm/pull/21004)
- Guardrail Edit: LiteLLM Content Filter Categories - [PR #21002](https://github.com/BerriAI/litellm/pull/21002)
- Add null guard for models in API keys table - [PR #20655](https://github.com/BerriAI/litellm/pull/20655)
- Show error details instead of 'Data Not Available' for failed requests - [PR #20656](https://github.com/BerriAI/litellm/pull/20656)
- Fix Spend Management Tests - [PR #21088](https://github.com/BerriAI/litellm/pull/21088)
- Fix JWT email domain validation error message - [PR #21212](https://github.com/BerriAI/litellm/pull/21212)
---
## AI Integrations
### Logging
- **[PostHog](../../docs/observability/posthog_integration)**
- Fix JSON serialization error for non-serializable objects - [PR #20668](https://github.com/BerriAI/litellm/pull/20668)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Sanitize label values to prevent metric scrape failures - [PR #20600](https://github.com/BerriAI/litellm/pull/20600)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Prevent empty proxy request spans from being sent to Langfuse - [PR #19935](https://github.com/BerriAI/litellm/pull/19935)
- **[OpenTelemetry](../../docs/proxy/logging#otel)**
- Auto-infer `otlp_http` exporter when endpoint is configured - [PR #20438](https://github.com/BerriAI/litellm/pull/20438)
- **[CloudZero](../../docs/proxy/logging)**
- Update CBF field mappings per LIT-1907 - [PR #20906](https://github.com/BerriAI/litellm/pull/20906)
- **General**
- Allow `MAX_CALLBACKS` override via env var - [PR #20781](https://github.com/BerriAI/litellm/pull/20781)
- Add `standard_logging_payload_excluded_fields` config option - [PR #20831](https://github.com/BerriAI/litellm/pull/20831)
- Enable `verbose_logger` when `LITELLM_LOG=DEBUG` - [PR #20496](https://github.com/BerriAI/litellm/pull/20496)
- Guard against None `litellm_metadata` in batch logging path - [PR #20832](https://github.com/BerriAI/litellm/pull/20832)
- Propagate model-level tags from config to SpendLogs - [PR #20769](https://github.com/BerriAI/litellm/pull/20769)
### Guardrails
- **Policy Templates**
- New Policy Templates: pre-configured guardrail combinations for specific use-cases - [PR #21025](https://github.com/BerriAI/litellm/pull/21025)
- Add NSFW policy template, toxic keywords in multiple languages, child safety content filter, JSON content viewer - [PR #21205](https://github.com/BerriAI/litellm/pull/21205)
- Add toxic/abusive content filter guardrails - [PR #20934](https://github.com/BerriAI/litellm/pull/20934)
- **Pipeline Execution**
- Add guardrail pipeline support for conditional sequential execution - [PR #21177](https://github.com/BerriAI/litellm/pull/21177)
- Agent Guardrails on streaming output - [PR #21206](https://github.com/BerriAI/litellm/pull/21206)
- Pipeline flow builder UI - [PR #21188](https://github.com/BerriAI/litellm/pull/21188)
- **[Zscaler AI Guard](../../docs/apply_guardrail)**
- Zscaler AI Guard bug fixes and support during post-call - [PR #20801](https://github.com/BerriAI/litellm/pull/20801)
- Zscaler AI Guard UI - [PR #21077](https://github.com/BerriAI/litellm/pull/21077)
- **[ZGuard](../../docs/apply_guardrail)**
- Add team policy mapping for ZGuard - [PR #20608](https://github.com/BerriAI/litellm/pull/20608)
- **General**
- Add logging to all unified guardrails + link to custom code guardrail templates - [PR #20900](https://github.com/BerriAI/litellm/pull/20900)
- Forward request headers + `litellm_version` to generic guardrails - [PR #20729](https://github.com/BerriAI/litellm/pull/20729)
- Empty `guardrails`/`policies` arrays should not trigger enterprise license check - [PR #20567](https://github.com/BerriAI/litellm/pull/20567)
- Fix OpenAI moderation guardrails - [PR #20718](https://github.com/BerriAI/litellm/pull/20718)
- Fix `/v2/guardrails/list` returning sensitive values - [PR #20796](https://github.com/BerriAI/litellm/pull/20796)
- Fix guardrail status error - [PR #20972](https://github.com/BerriAI/litellm/pull/20972)
- Reuse `get_instance_fn` in `initialize_custom_guardrail` - [PR #20917](https://github.com/BerriAI/litellm/pull/20917)
---
## Spend Tracking, Budgets and Rate Limiting
- **Prevent shared backend model key from being polluted** by per-deployment custom pricing - [PR #20679](https://github.com/BerriAI/litellm/pull/20679)
- **Avoid in-place mutation** in SpendUpdateQueue aggregation - [PR #20876](https://github.com/BerriAI/litellm/pull/20876)
---
## MCP Gateway (12 updates)
- **MCP M2M OAuth2 Support** - Add support for machine-to-machine OAuth2 for MCP servers - [PR #20788](https://github.com/BerriAI/litellm/pull/20788)
- **MCP Server Discovery UI** - Browse and discover available MCP servers from the UI - [PR #21079](https://github.com/BerriAI/litellm/pull/21079)
- **MCP Tracing** - Add OpenTelemetry tracing for MCP calls running through AI Gateway - [PR #21018](https://github.com/BerriAI/litellm/pull/21018)
- **MCP OAuth2 Debug Headers** - Client-side debug headers for OAuth2 troubleshooting - [PR #21151](https://github.com/BerriAI/litellm/pull/21151)
- **Fix MCP "Session not found" errors** - Resolve session persistence issues - [PR #21040](https://github.com/BerriAI/litellm/pull/21040)
- **Fix MCP OAuth2 root endpoints** returning "MCP server not found" - [PR #20784](https://github.com/BerriAI/litellm/pull/20784)
- **Fix MCP OAuth2 query param merging** when `authorization_url` already contains params - [PR #20968](https://github.com/BerriAI/litellm/pull/20968)
- **Fix MCP SCOPES on Atlassian** issue - [PR #21150](https://github.com/BerriAI/litellm/pull/21150)
- **Fix MCP StreamableHTTP backend** - Use `anyio.fail_after` instead of `asyncio.wait_for` - [PR #20891](https://github.com/BerriAI/litellm/pull/20891)
- **Inject `NPM_CONFIG_CACHE`** into STDIO MCP subprocess env - [PR #21069](https://github.com/BerriAI/litellm/pull/21069)
- **Block spaces and hyphens** in MCP server names and aliases - [PR #21074](https://github.com/BerriAI/litellm/pull/21074)
---
## Performance / Loadbalancing / Reliability improvements (8 improvements)
- **Remove orphan entries from queue** - Fix memory leak in scheduler queue - [PR #20866](https://github.com/BerriAI/litellm/pull/20866)
- **Remove repeated provider parsing** in budget limiter hot path - [PR #21043](https://github.com/BerriAI/litellm/pull/21043)
- **Use current retry exception** for retry backoff instead of stale exception - [PR #20725](https://github.com/BerriAI/litellm/pull/20725)
- **Add Semgrep & fix OOMs** - Static analysis rules and out-of-memory fixes - [PR #20912](https://github.com/BerriAI/litellm/pull/20912)
- **Add Pyroscope** for continuous profiling and observability - [PR #21167](https://github.com/BerriAI/litellm/pull/21167)
- **Respect `ssl_verify`** with shared aiohttp sessions - [PR #20349](https://github.com/BerriAI/litellm/pull/20349)
- **Fix shared health check serialization** - [PR #21119](https://github.com/BerriAI/litellm/pull/21119)
- **Change model mismatch logs** from WARNING to DEBUG - [PR #20994](https://github.com/BerriAI/litellm/pull/20994)
---
## Database Changes
### Schema Updates
| Table | Change Type | Description | PR | Migration |
| ----- | ----------- | ----------- | -- | --------- |
| `LiteLLM_VerificationToken` | New Indexes | Added indexes on `user_id`+`team_id`, `team_id`, and `budget_reset_at`+`expires` | [PR #20736](https://github.com/BerriAI/litellm/pull/20736) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260209085821_add_verificationtoken_indexes/migration.sql) |
| `LiteLLM_PolicyAttachmentTable` | New Column | Added `tags` text array for policy-to-tag connections | [PR #21061](https://github.com/BerriAI/litellm/pull/21061) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql) |
| `LiteLLM_AccessGroupTable` | New Table | Access groups for managing model, MCP server, and agent access | [PR #21022](https://github.com/BerriAI/litellm/pull/21022) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql) |
| `LiteLLM_AccessGroupTable` | Column Change | Renamed `access_model_ids` to `access_model_names` | [PR #21166](https://github.com/BerriAI/litellm/pull/21166) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260213170952_access_group_change_to_model_name/migration.sql) |
| `LiteLLM_ManagedVectorStoreTable` | New Table | Managed vector store tracking with model mappings | - | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260213105436_add_managed_vector_store_table/migration.sql) |
| `LiteLLM_TeamTable`, `LiteLLM_VerificationToken` | New Column | Added `access_group_ids` text array | [PR #21022](https://github.com/BerriAI/litellm/pull/21022) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql) |
| `LiteLLM_GuardrailsTable` | New Column | Added `team_id` text column | - | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260214094754_schema_sync/migration.sql) |
---
## Documentation Updates (14 updates)
- LiteLLM Observatory section added to v1.81.9 release notes - [PR #20675](https://github.com/BerriAI/litellm/pull/20675)
- Callback registration optimization added to release notes - [PR #20681](https://github.com/BerriAI/litellm/pull/20681)
- Middleware performance blog post - [PR #20677](https://github.com/BerriAI/litellm/pull/20677)
- UI Team Soft Budget documentation - [PR #20669](https://github.com/BerriAI/litellm/pull/20669)
- UI Contributing and Troubleshooting guide - [PR #20674](https://github.com/BerriAI/litellm/pull/20674)
- Reorganize Admin UI subsection - [PR #20676](https://github.com/BerriAI/litellm/pull/20676)
- SDK proxy authentication (OAuth2/JWT auto-refresh) - [PR #20680](https://github.com/BerriAI/litellm/pull/20680)
- Forward client headers to LLM API documentation fix - [PR #20768](https://github.com/BerriAI/litellm/pull/20768)
- Add docs guide for using policies - [PR #20914](https://github.com/BerriAI/litellm/pull/20914)
- Add native thinking param examples for Claude Opus 4.6 - [PR #20799](https://github.com/BerriAI/litellm/pull/20799)
- Fix Claude Code MCP tutorial - [PR #21145](https://github.com/BerriAI/litellm/pull/21145)
- Add API base URLs for Dashscope (International and China/Beijing) - [PR #21083](https://github.com/BerriAI/litellm/pull/21083)
- Fix `DEFAULT_NUM_WORKERS_LITELLM_PROXY` default (1, not 4) - [PR #21127](https://github.com/BerriAI/litellm/pull/21127)
- Correct ElevenLabs support status in README - [PR #20643](https://github.com/BerriAI/litellm/pull/20643)
---
## New Contributors
* @iver56 made their first contribution in [PR #20643](https://github.com/BerriAI/litellm/pull/20643)
* @eliasaronson made their first contribution in [PR #20666](https://github.com/BerriAI/litellm/pull/20666)
* @NirantK made their first contribution in [PR #19656](https://github.com/BerriAI/litellm/pull/19656)
* @looksgood made their first contribution in [PR #20919](https://github.com/BerriAI/litellm/pull/20919)
* @kelvin-tran made their first contribution in [PR #20548](https://github.com/BerriAI/litellm/pull/20548)
* @bluet made their first contribution in [PR #20873](https://github.com/BerriAI/litellm/pull/20873)
* @itayov made their first contribution in [PR #20729](https://github.com/BerriAI/litellm/pull/20729)
* @CSteigstra made their first contribution in [PR #20960](https://github.com/BerriAI/litellm/pull/20960)
* @rahulrd25 made their first contribution in [PR #20569](https://github.com/BerriAI/litellm/pull/20569)
* @muraliavarma made their first contribution in [PR #20598](https://github.com/BerriAI/litellm/pull/20598)
* @joaokopernico made their first contribution in [PR #21039](https://github.com/BerriAI/litellm/pull/21039)
* @datzscaler made their first contribution in [PR #21077](https://github.com/BerriAI/litellm/pull/21077)
* @atapia27 made their first contribution in [PR #20922](https://github.com/BerriAI/litellm/pull/20922)
* @fpagny made their first contribution in [PR #21121](https://github.com/BerriAI/litellm/pull/21121)
* @aidankovacic-8451 made their first contribution in [PR #21119](https://github.com/BerriAI/litellm/pull/21119)
* @luisgallego-aily made their first contribution in [PR #19935](https://github.com/BerriAI/litellm/pull/19935)
---
## Full Changelog
[v1.81.9.rc.1...v1.81.12.rc.1](https://github.com/BerriAI/litellm/compare/v1.81.9.rc.1...v1.81.12.rc.1)

8
docs/my-website/release_notes/v1.81.6.md
View file

@ -14,6 +14,14 @@ authors:
hide_table_of_contents: false
---
:::danger Known Issue - CPU Usage
This release had known issues with CPU usage. This has been fixed in [v1.81.9-stable](./v1-81-9).
**We recommend using v1.81.9-stable instead.**
:::
## Deploy this version
import Tabs from '@theme/Tabs';

14
docs/my-website/release_notes/v1.81.9.md
View file

@ -1,5 +1,5 @@
---
title: "[Preview] v1.81.9 - Control which MCP Servers are exposed on the Internet"
title: "v1.81.9 - Control which MCP Servers are exposed on the Internet"
slug: "v1-81-9"
date: 2026-02-07T00:00:00
authors:
@ -14,6 +14,16 @@ authors:
hide_table_of_contents: false
---
:::info Stable Release Branch
For each stable release, we now maintain a dedicated branch with the format `litellm_stable_release_branch_x_xx_xx` for the version.
This allows easier patching for day 0 model launches.
**Branch for v1.81.9:** [litellm_stable_release_branch_1_81_9](https://github.com/BerriAI/litellm/tree/litellm_stable_release_branch_1_81_9)
:::
## Deploy this version
import Tabs from '@theme/Tabs';
@ -27,7 +37,7 @@ import Image from '@theme/IdealImage';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.81.9.rc.1
ghcr.io/berriai/litellm:main-v1.81.9-stable
```
</TabItem>

29
docs/my-website/sidebars.js
View file

@ -97,6 +97,7 @@ const sidebars = {
label: "Policies",
items: [
"proxy/guardrails/guardrail_policies",
"proxy/guardrails/policy_templates",
"proxy/guardrails/policy_tags",
],
},
@ -106,7 +107,8 @@ const sidebars = {
items: [
"proxy/alerting",
"proxy/pagerduty",
"proxy/prometheus"
"proxy/prometheus",
"proxy/pyroscope_profiling"
]
},
{
@ -465,6 +467,7 @@ const sidebars = {
"proxy/model_access_guide",
"proxy/model_access",
"proxy/model_access_groups",
"proxy/access_groups",
"proxy/team_model_add"
]
},
@ -873,6 +876,7 @@ const sidebars = {
},
"providers/sambanova",
"providers/sap",
"providers/scaleway",
"providers/stability",
"providers/synthetic",
"providers/snowflake",
@ -1002,6 +1006,7 @@ const sidebars = {
"tutorials/presidio_pii_masking",
"tutorials/elasticsearch_logging",
"tutorials/gemini_realtime_with_audio",
"tutorials/claude_code_beta_headers",
{
type: "category",
label: "LiteLLM Python SDK Tutorials",
@ -1096,16 +1101,24 @@ const sidebars = {
"proxy_server",
],
},
"troubleshoot",
{
type: "category",
label: "Issue Reporting",
label: "Troubleshooting",
items: [
"troubleshoot/prisma_migrations",
"troubleshoot/cpu_issues",
"troubleshoot/memory_issues",
"troubleshoot/spend_queue_warnings",
"troubleshoot/max_callbacks",
"troubleshoot/ui_issues",
"mcp_troubleshoot",
{
type: "category",
label: "Performance / Latency",
items: [
"troubleshoot/cpu_issues",
"troubleshoot/memory_issues",
"troubleshoot/spend_queue_warnings",
"troubleshoot/max_callbacks",
"troubleshoot/prisma_migrations",
],
},
"troubleshoot",
],
},
{

BIN
enterprise/dist/litellm_enterprise-0.1.32-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.32.tar.gz vendored Normal file
View file

Binary file not shown.

5
enterprise/enterprise_hooks/__init__.py
View file

@ -1,11 +1,15 @@
from typing import Dict, Literal, Type, Union
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
from litellm_enterprise.proxy.hooks.managed_vector_stores import (
_PROXY_LiteLLMManagedVectorStores,
)
from litellm.integrations.custom_logger import CustomLogger
ENTERPRISE_PROXY_HOOKS: Dict[str, Type[CustomLogger]] = {
"managed_files": _PROXY_LiteLLMManagedFiles,
"managed_vector_stores": _PROXY_LiteLLMManagedVectorStores,
}
@ -13,6 +17,7 @@ def get_enterprise_proxy_hook(
hook_name: Union[
Literal[
"managed_files",
"managed_vector_stores",
"max_parallel_requests",
],
str,

5
enterprise/litellm_enterprise/proxy/auth/route_checks.py
View file

@ -41,6 +41,10 @@ class EnterpriseRouteChecks:
return get_secret_bool("DISABLE_ADMIN_ENDPOINTS") is True
# Routes that should remain accessible even when LLM API endpoints are disabled.
# These are read-only model listing routes needed by the Admin UI.
LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"]
@staticmethod
def should_call_route(route: str):
"""
@ -58,6 +62,7 @@ class EnterpriseRouteChecks:
)
elif (
RouteChecks.is_llm_api_route(route=route)
and route not in EnterpriseRouteChecks.LLM_API_EXEMPT_ROUTES
and EnterpriseRouteChecks.is_llm_api_route_disabled()
):
raise HTTPException(

464
enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py Normal file
View file

@ -0,0 +1,464 @@
# What is this?
## This hook is used to manage vector stores with target_model_names support
## It allows creating vector stores across multiple models and managing them with unified IDs
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
from fastapi import HTTPException
import litellm
from litellm import Router, verbose_logger
from litellm._uuid import uuid
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.base_llm.managed_resources import BaseManagedResource
from litellm.llms.base_llm.managed_resources.utils import (
generate_unified_id_string,
is_base64_encoded_unified_id,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.vector_stores import (
VectorStoreCreateOptionalRequestParams,
VectorStoreCreateResponse,
)
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
Span = Any
InternalUsageCache = Any
PrismaClient = Any
class _PROXY_LiteLLMManagedVectorStores(
CustomLogger, BaseManagedResource[VectorStoreCreateResponse]
):
"""
Managed vector stores with target_model_names support.
This class provides functionality to:
- Create vector stores across multiple models
- Retrieve vector stores by unified ID
- Delete vector stores from all models
- List vector stores created by a user
"""
def __init__(
self, internal_usage_cache: InternalUsageCache, prisma_client: PrismaClient
):
CustomLogger.__init__(self)
BaseManagedResource.__init__(self, internal_usage_cache, prisma_client)
# ============================================================================
# ABSTRACT METHOD IMPLEMENTATIONS
# ============================================================================
@property
def resource_type(self) -> str:
"""Return the resource type identifier."""
return "vector_store"
@property
def table_name(self) -> str:
"""Return the database table name for vector stores."""
# Prisma converts model name LiteLLM_ManagedVectorStoreTable to litellm_managedvectorstoretable
return "litellm_managedvectorstoretable"
def get_unified_resource_id_format(
self,
resource_object: VectorStoreCreateResponse,
target_model_names_list: List[str],
) -> str:
"""
Generate the format string for the unified vector store ID.
Format:
litellm_proxy:vector_store;unified_id,<uuid>;target_model_names,<models>;resource_id,<vs_id>;model_id,<model_id>
"""
# VectorStoreCreateResponse is a TypedDict, so resource_object is a dictionary
# Extract provider resource ID from the response
provider_resource_id = resource_object.get("id", "")
# Model ID is stored in hidden params if the response object supports it
# For TypedDict responses, we need to check if _hidden_params was added
hidden_params: Dict[str, Any] = {}
if hasattr(resource_object, "_hidden_params"):
hidden_params = getattr(resource_object, "_hidden_params", {}) or {}
model_id = hidden_params.get("model_id", "")
return generate_unified_id_string(
resource_type=self.resource_type,
unified_uuid=str(uuid.uuid4()),
target_model_names=target_model_names_list,
provider_resource_id=provider_resource_id,
model_id=model_id,
)
async def create_resource_for_model(
self,
llm_router: Router,
model: str,
request_data: Dict[str, Any],
litellm_parent_otel_span: Span,
) -> VectorStoreCreateResponse:
"""
Create a vector store for a specific model.
Args:
llm_router: LiteLLM router instance
model: Model name to create vector store for
request_data: Request data for vector store creation
litellm_parent_otel_span: OpenTelemetry span for tracing
Returns:
VectorStoreCreateResponse from the provider
"""
# Use the router to create the vector store
response = await llm_router.avector_store_create(
model=model, **request_data
)
return response
# ============================================================================
# VECTOR STORE CRUD OPERATIONS
# ============================================================================
async def acreate_vector_store(
self,
create_request: VectorStoreCreateOptionalRequestParams,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
user_api_key_dict: UserAPIKeyAuth,
) -> VectorStoreCreateResponse:
"""
Create a vector store across multiple models.
Args:
create_request: Vector store creation request parameters
llm_router: LiteLLM router instance
target_model_names_list: List of target model names
litellm_parent_otel_span: OpenTelemetry span for tracing
user_api_key_dict: User API key authentication details
Returns:
VectorStoreCreateResponse with unified ID
"""
verbose_logger.info(
f"Creating managed vector store for models: {target_model_names_list}"
)
# Create vector store for each model
# Convert TypedDict to Dict[str, Any] for base class compatibility
request_data_dict: Dict[str, Any] = dict(create_request)
responses = await self.create_resource_for_each_model(
llm_router=llm_router,
request_data=request_data_dict,
target_model_names_list=target_model_names_list,
litellm_parent_otel_span=litellm_parent_otel_span,
)
# Generate unified ID
unified_id = self.generate_unified_resource_id(
resource_objects=responses,
target_model_names_list=target_model_names_list,
)
# Extract model mappings from responses
model_mappings: Dict[str, str] = {}
for response in responses:
hidden_params = getattr(response, "_hidden_params", {}) or {}
model_id = hidden_params.get("model_id")
if model_id:
# VectorStoreCreateResponse is a TypedDict, use dict access
model_mappings[model_id] = response["id"]
verbose_logger.debug(
f"Created vector stores with model mappings: {model_mappings}"
)
# Store in database
await self.store_unified_resource_id(
unified_resource_id=unified_id,
resource_object=responses[0], # Store first response as template
litellm_parent_otel_span=litellm_parent_otel_span,
model_mappings=model_mappings,
user_api_key_dict=user_api_key_dict,
)
# Return response with unified ID
# VectorStoreCreateResponse is a TypedDict, so we need to create a new dict with the unified ID
response = responses[0].copy()
response["id"] = unified_id
verbose_logger.info(
f"Successfully created managed vector store with unified ID: {unified_id}"
)
return response
async def alist_vector_stores(
self,
user_api_key_dict: UserAPIKeyAuth,
limit: Optional[int] = None,
after: Optional[str] = None,
order: Optional[str] = None,
) -> Dict[str, Any]:
"""
List vector stores created by a user.
Args:
user_api_key_dict: User API key authentication details
limit: Maximum number of vector stores to return
after: Cursor for pagination
order: Sort order ('asc' or 'desc')
Returns:
Dictionary with list of vector stores and pagination info
"""
# Use the base class method
return await self.list_user_resources(
user_api_key_dict=user_api_key_dict,
limit=limit,
after=after,
)
# ============================================================================
# ACCESS CONTROL
# ============================================================================
async def check_vector_store_access(
self, vector_store_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
"""
Check if user has access to a vector store.
Args:
vector_store_id: The unified vector store ID
user_api_key_dict: User API key authentication details
Returns:
True if user has access, False otherwise
"""
is_unified_id = is_base64_encoded_unified_id(vector_store_id)
if is_unified_id:
# Check access for managed vector store
return await self.can_user_access_unified_resource_id(
vector_store_id,
user_api_key_dict,
)
# Not a managed vector store, allow access
return True
async def check_managed_vector_store_access(
self, data: Dict, user_api_key_dict: UserAPIKeyAuth
) -> bool:
"""
Check if user has access to a managed vector store in request data.
Args:
data: Request data containing vector_store_id
user_api_key_dict: User API key authentication details
Returns:
True if this is a managed vector store and user has access
Raises:
HTTPException: If user doesn't have access
"""
vector_store_id = cast(Optional[str], data.get("vector_store_id"))
is_unified_id = (
is_base64_encoded_unified_id(vector_store_id)
if vector_store_id
else False
)
if is_unified_id and vector_store_id:
if await self.can_user_access_unified_resource_id(
vector_store_id, user_api_key_dict
):
return True
else:
raise HTTPException(
status_code=403,
detail=f"User {user_api_key_dict.user_id} does not have access to vector store {vector_store_id}",
)
return False
# ============================================================================
# PRE-CALL HOOK (For Router Integration)
# ============================================================================
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: Any,
data: Dict,
call_type: str,
) -> Union[Exception, str, Dict, None]:
"""
Pre-call hook to handle vector store operations.
This hook intercepts vector store requests and:
- Validates access for managed vector stores
- Transforms unified IDs to provider-specific IDs
- Adds model routing information
Args:
user_api_key_dict: User API key authentication details
cache: Cache instance
data: Request data
call_type: Type of call being made
Returns:
Modified request data or None
"""
from litellm.llms.base_llm.managed_resources.utils import (
is_base64_encoded_unified_id,
parse_unified_id,
)
# Handle vector store search operations
if call_type == "avector_store_search":
vector_store_id = data.get("vector_store_id")
if vector_store_id:
# Check if it's a managed vector store ID
decoded_id = is_base64_encoded_unified_id(vector_store_id)
if decoded_id:
verbose_logger.debug(
f"Processing managed vector store search: {vector_store_id}"
)
# Check access
has_access = await self.can_user_access_unified_resource_id(
vector_store_id, user_api_key_dict
)
if not has_access:
raise HTTPException(
status_code=403,
detail=f"User {user_api_key_dict.user_id} does not have access to vector store {vector_store_id}",
)
# Parse the unified ID to extract components
parsed_id = parse_unified_id(vector_store_id)
if parsed_id:
# Extract the model ID and provider resource ID
model_id = parsed_id.get("model_id")
provider_resource_id = parsed_id.get("provider_resource_id")
target_model_names = parsed_id.get("target_model_names", [])
verbose_logger.debug(
f"Decoded vector store - model_id: {model_id}, provider_resource_id: {provider_resource_id}, target_model_names: {target_model_names}"
)
# Determine which model to use for routing
# Priority: model_id (deployment ID) > first target_model_name
routing_model = None
if model_id:
routing_model = model_id
elif target_model_names and len(target_model_names) > 0:
routing_model = target_model_names[0]
# Set the model for routing
if routing_model:
data["model"] = routing_model
verbose_logger.info(
f"Routing vector store search to model: {routing_model}"
)
# Replace the unified ID with the provider-specific ID
if provider_resource_id:
data["vector_store_id"] = provider_resource_id
verbose_logger.debug(
f"Replaced unified ID with provider resource ID: {provider_resource_id}"
)
# Handle vector store retrieve/delete operations
elif call_type in ("avector_store_retrieve", "avector_store_delete"):
await self.check_managed_vector_store_access(data, user_api_key_dict)
# If it's a managed vector store, we'll handle it in the endpoint
# No need to transform here as the endpoint will route to the hook
return data
# ============================================================================
# POST-CALL HOOK (For Response Transformation)
# ============================================================================
async def async_post_call_success_hook(
self,
data: Dict,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
) -> Any:
"""
Post-call hook to transform responses.
This hook can be used to transform responses if needed.
For now, it just passes through the response.
Args:
data: Request data
user_api_key_dict: User API key authentication details
response: Response from the provider
Returns:
Potentially modified response
"""
# Currently no transformation needed
return response
# ============================================================================
# DEPLOYMENT FILTERING
# ============================================================================
async def async_filter_deployments( # type: ignore[override]
self,
model: str,
healthy_deployments: List,
messages: Optional[List] = None,
request_kwargs: Optional[Dict] = None,
parent_otel_span: Optional[Span] = None,
) -> List[Dict]:
"""
Filter deployments based on vector store availability.
This is used by the router to select only deployments that have
the vector store available.
Note: This method signature is a compromise between CustomLogger and BaseManagedResource
parent classes which have incompatible signatures. The type: ignore[override] is necessary
due to this multiple inheritance conflict.
Args:
model: Model name
healthy_deployments: List of healthy deployments
messages: Messages (unused for vector stores, required by CustomLogger interface)
request_kwargs: Request kwargs containing vector_store_id and mappings
parent_otel_span: OpenTelemetry span for tracing
Returns:
Filtered list of deployments
"""
return await BaseManagedResource.async_filter_deployments(
self,
model=model,
healthy_deployments=healthy_deployments,
request_kwargs=request_kwargs,
parent_otel_span=parent_otel_span,
resource_id_key="vector_store_id",
)

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.31"
version = "0.1.32"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.31"
version = "0.1.32"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",

127
litellm-proxy-extras/build_and_publish.md Normal file
View file

@ -0,0 +1,127 @@
# Build & Publish `litellm-proxy-extras`
This runbook covers building and publishing a new version of the `litellm-proxy-extras` PyPI package. For use by litellm engineers only.
## Prerequisites
- All `schema.prisma` files are in sync (see [migration_runbook.md](./migration_runbook.md) Step 0)
- Migration has been generated and committed
- You are in the `litellm-proxy-extras/` directory
## Step 1: Bump the Version
### Option A: Automatic Version Bump (Recommended)
Use commitizen to automatically bump the version across all files:
```bash
cd litellm-proxy-extras
cz bump --increment patch
```
This will automatically:
- Bump the version in `pyproject.toml` (both `[tool.poetry].version` and `[tool.commitizen].version`)
- Update the version in `../requirements.txt`
- Update the version in `../pyproject.toml` (root)
- Create a git commit with the version bump
Then skip to Step 3 (Install Build Dependencies).
### Option B: Manual Version Bump
Update the version in `pyproject.toml`:
```bash
cd litellm-proxy-extras
# Check current version
grep 'version' pyproject.toml
```
Edit `pyproject.toml` and bump the version (both `[tool.poetry].version` and `[tool.commitizen].version`).
#### Step 2: Update Version in Root Package Files (Manual Only)
After bumping the version in `litellm-proxy-extras/pyproject.toml`, you **must** also update the version reference in the root-level files:
| File | Line to update |
|------|---------------|
| `requirements.txt` | `litellm-proxy-extras==X.Y.Z` |
| `pyproject.toml` (root) | `litellm-proxy-extras = {version = "X.Y.Z", optional = true}` |
```bash
# From the repo root — replace OLD with NEW version
sed -i '' 's/litellm-proxy-extras==OLD/litellm-proxy-extras==NEW/' requirements.txt
sed -i '' 's/litellm-proxy-extras = {version = "OLD"/litellm-proxy-extras = {version = "NEW"/' pyproject.toml
```
> **Do NOT skip this step.** The main `litellm` package pins the extras version — if you don't update these, users will install the old version.
## Step 3: Install Build Dependencies
```bash
pip install build twine
```
## Step 4: Clean Old Artifacts
```bash
rm -rf dist/ build/ *.egg-info
```
## Step 5: Build the Package
```bash
python3 -m build
```
This creates `.tar.gz` and `.whl` files in the `dist/` directory.
Verify the build output:
```bash
ls -la dist/
```
## Step 6: Upload to PyPI
```bash
twine upload dist/*
```
You will be prompted for your PyPI API token:
```
Enter your API token: pypi-...
```
> Use `__token__` as the username and your PyPI API token as the password.
## Quick Reference (Copy-Paste)
```bash
cd litellm-proxy-extras
rm -rf dist/ build/ *.egg-info
python3 -m build
twine upload dist/*
```
---
## Do you want to build and publish a new `litellm-proxy-extras` package? (y/n)
If **yes**, run the following commands in order:
```bash
cd litellm-proxy-extras
pip install build twine
rm -rf dist/ build/ *.egg-info
python3 -m build
twine upload dist/*
```
When `twine upload` runs, enter your PyPI credentials:
- **Username:** `__token__`
- **Password:** *(paste your PyPI API key)*
If **no**, you're done — no package publish needed.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.35.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.36.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.37-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.37.tar.gz vendored Normal file
View file

Binary file not shown.

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260212103349_adjust_tags_policy_table/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_PolicyAttachmentTable" ADD COLUMN "tags" TEXT[] DEFAULT ARRAY[]::TEXT[];

33
litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql Normal file
View file

@ -0,0 +1,33 @@
-- AlterTable
ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[];
-- AlterTable
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[];
-- AlterTable
ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[];
-- AlterTable
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[];
-- CreateTable
CREATE TABLE "LiteLLM_AccessGroupTable" (
"access_group_id" TEXT NOT NULL,
"access_group_name" TEXT NOT NULL,
"description" TEXT,
"access_model_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
"access_mcp_server_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
"access_agent_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
"assigned_team_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
"assigned_key_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT,
CONSTRAINT "LiteLLM_AccessGroupTable_pkey" PRIMARY KEY ("access_group_id")
);
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_AccessGroupTable_access_group_name_key" ON "LiteLLM_AccessGroupTable"("access_group_name");

22
litellm-proxy-extras/litellm_proxy_extras/migrations/20260213105436_add_managed_vector_store_table/migration.sql Normal file
View file

@ -0,0 +1,22 @@
-- CreateTable
CREATE TABLE "LiteLLM_ManagedVectorStoreTable" (
"id" TEXT NOT NULL,
"unified_resource_id" TEXT NOT NULL,
"resource_object" JSONB,
"model_mappings" JSONB NOT NULL,
"flat_model_resource_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
"storage_backend" TEXT,
"storage_url" TEXT,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT,
"updated_at" TIMESTAMP(3) NOT NULL,
"updated_by" TEXT,
CONSTRAINT "LiteLLM_ManagedVectorStoreTable_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_ManagedVectorStoreTable_unified_resource_id_key" ON "LiteLLM_ManagedVectorStoreTable"("unified_resource_id");
-- CreateIndex
CREATE INDEX "LiteLLM_ManagedVectorStoreTable_unified_resource_id_idx" ON "LiteLLM_ManagedVectorStoreTable"("unified_resource_id");

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260213170952_access_group_change_to_model_name/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_AccessGroupTable" DROP COLUMN "access_model_ids",
ADD COLUMN "access_model_names" TEXT[] DEFAULT ARRAY[]::TEXT[];

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260214094754_schema_sync/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_GuardrailsTable" ADD COLUMN "team_id" TEXT;

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260214163027_add_pipeline_to_policy_table/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_PolicyTable" ADD COLUMN "pipeline" JSONB;

41
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -128,6 +128,7 @@ model LiteLLM_TeamTable {
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
team_member_permissions String[] @default([])
access_group_ids String[] @default([])
policies String[] @default([])
model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases
allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team
@ -161,6 +162,7 @@ model LiteLLM_DeletedTeamTable {
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
team_member_permissions String[] @default([])
access_group_ids String[] @default([])
policies String[] @default([])
model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases
allow_team_guardrail_config Boolean @default(false)
@ -293,6 +295,7 @@ model LiteLLM_VerificationToken {
allowed_cache_controls String[] @default([])
allowed_routes String[] @default([])
policies String[] @default([])
access_group_ids String[] @default([])
model_spend Json @default("{}")
model_max_budget Json @default("{}")
budget_id String?
@ -348,6 +351,7 @@ model LiteLLM_DeletedVerificationToken {
allowed_cache_controls String[] @default([])
allowed_routes String[] @default([])
policies String[] @default([])
access_group_ids String[] @default([])
model_spend Json @default("{}")
model_max_budget Json @default("{}")
router_settings Json? @default("{}")
@ -766,6 +770,22 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t
@@index([model_object_id])
}
model LiteLLM_ManagedVectorStoreTable {
id String @id @default(uuid())
unified_resource_id String @unique // The base64 encoded unified vector store ID
resource_object Json? // Stores the VectorStoreCreateResponse
model_mappings Json // Maps model_id -> provider_vector_store_id
flat_model_resource_ids String[] @default([]) // Flat list of provider vector store IDs for faster querying
storage_backend String? // Storage backend name (if applicable)
storage_url String? // Storage URL (if applicable)
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_resource_id])
}
model LiteLLM_ManagedVectorStoresTable {
vector_store_id String @id
custom_llm_provider String
@ -900,6 +920,7 @@ model LiteLLM_PolicyTable {
guardrails_add String[] @default([])
guardrails_remove String[] @default([])
condition Json? @default("{}") // Policy conditions (e.g., model matching)
pipeline Json? // Optional guardrail pipeline (mode + steps[])
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
@ -920,3 +941,23 @@ model LiteLLM_PolicyAttachmentTable {
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}
//Unified Access Groups table for storing unified access groups
model LiteLLM_AccessGroupTable {
access_group_id String @id @default(uuid())
access_group_name String @unique
description String?
// Resource memberships - explicit arrays per type
access_model_names String[] @default([])
access_mcp_server_ids String[] @default([])
access_agent_ids String[] @default([])
assigned_team_ids String[] @default([])
assigned_key_ids String[] @default([])
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}

37
litellm-proxy-extras/migration_runbook.md
View file

@ -2,7 +2,35 @@
This is a runbook for creating and running database migrations for the LiteLLM proxy. For use for litellm engineers only.
## Quick Start
## Step 0: Sync All `schema.prisma` Files
Before doing anything else, make sure all `schema.prisma` files in the repo are in sync. There are multiple copies that must match:
| File | Purpose |
|------|---------|
| `schema.prisma` (repo root) | Source of truth |
| `litellm/proxy/schema.prisma` | Used by the proxy server |
| `litellm-proxy-extras/litellm_proxy_extras/schema.prisma` | Used for migration generation |
**Sync process:**
```bash
# 1. Diff all schema files against the root source of truth
diff schema.prisma litellm/proxy/schema.prisma
diff schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma
# 2. If there are differences, copy the root schema to all locations
cp schema.prisma litellm/proxy/schema.prisma
cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma
# 3. Verify all files are now identical
diff schema.prisma litellm/proxy/schema.prisma && echo "proxy schema in sync" || echo "MISMATCH"
diff schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma && echo "extras schema in sync" || echo "MISMATCH"
```
> **Do NOT proceed to migration generation until all schema files are identical.**
## Step 1: Quick Start — Generate Migration
```bash
# Install deps (one time)
@ -43,8 +71,13 @@ rm -rf litellm-proxy-extras/litellm_proxy_extras/migrations/[empty_dir]
## Rules
- Update `schema.prisma` first
- Sync all `schema.prisma` files first (Step 0)
- Update `schema.prisma` at the repo root first, then sync copies
- Review generated SQL before committing
- Use descriptive migration names
- Never edit existing migration files
- Commit schema + migration together
---
**Done with migration?** See [build_and_publish.md](./build_and_publish.md) to publish a new `litellm-proxy-extras` package.

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.34"
version = "0.4.39"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.34"
version = "0.4.39"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

5
litellm/__init__.py
View file

@ -175,6 +175,7 @@ _async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # Custo
pre_call_rules: List[Callable] = []
post_call_rules: List[Callable] = []
turn_off_message_logging: Optional[bool] = False
standard_logging_payload_excluded_fields: Optional[List[str]] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it
log_raw_request_response: bool = False
redact_messages_in_exceptions: Optional[bool] = False
redact_user_api_key_info: Optional[bool] = False
@ -337,6 +338,10 @@ model_cost_map_url: str = os.getenv(
"LITELLM_MODEL_COST_MAP_URL",
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json",
)
anthropic_beta_headers_url: str = os.getenv(
"LITELLM_ANTHROPIC_BETA_HEADERS_URL",
"https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json",
)
suppress_debug_info = False
dynamodb_table_name: Optional[str] = None
s3_callback_params: Optional[Dict] = None

1
litellm/_lazy_imports_registry.py
View file

@ -275,7 +275,6 @@ LLM_CONFIG_NAMES = (
"LmStudioEmbeddingConfig",
"NscaleConfig",
"PerplexityChatConfig",
"PerplexityResponsesConfig",
"AzureOpenAIO1Config",
"IBMWatsonXAIConfig",
"IBMWatsonXChatConfig",

23
litellm/anthropic_beta_headers_config.json
View file

@ -2,8 +2,8 @@
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.",
"anthropic": {
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"bash_20241022": "bash_20241022",
"bash_20250124": "bash_20250124",
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": "compact-2026-01-12",
"computer-use-2025-01-24": "computer-use-2025-01-24",
@ -13,26 +13,27 @@
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": "fast-mode-2026-02-01",
"files-api-2025-04-14": "files-api-2025-04-14",
"structured-output-2024-03-01": "structured-output-2024-03-01",
"structured-output-2024-03-01": null,
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-servers-2025-12-04": "mcp-servers-2025-12-04",
"mcp-servers-2025-12-04": null,
"oauth-2025-04-20": "oauth-2025-04-20",
"output-128k-2025-02-19": "output-128k-2025-02-19",
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": "text_editor_20241022",
"text_editor_20250124": "text_editor_20250124",
"text_editor_20241022": null,
"text_editor_20250124": null,
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
"web-search-2025-03-05": "web-search-2025-03-05"
},
"azure_ai": {
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"bash_20241022": "bash_20241022",
"bash_20250124": "bash_20250124",
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": null,
"computer-use-2025-01-24": "computer-use-2025-01-24",
@ -46,7 +47,7 @@
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-servers-2025-12-04": "mcp-servers-2025-12-04",
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
@ -59,7 +60,7 @@
"web-search-2025-03-05": "web-search-2025-03-05"
},
"bedrock_converse": {
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
"advanced-tool-use-2025-11-20": null,
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": null,
@ -84,7 +85,7 @@
"text_editor_20241022": null,
"text_editor_20250124": null,
"token-efficient-tools-2025-02-19": null,
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"tool-search-tool-2025-10-19": null,
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": null
},

182
litellm/anthropic_beta_headers_manager.py
View file

@ -5,28 +5,167 @@ This module provides utilities to:
1. Load beta header configuration from JSON (mapping of supported headers per provider)
2. Filter and map beta headers based on provider support
3. Handle provider-specific header name mappings (e.g., advanced-tool-use -> tool-search-tool)
4. Support remote fetching and caching similar to model cost map
Design:
- JSON config contains mapping of beta headers for each provider
- Keys are input header names, values are provider-specific header names (or null if unsupported)
- Only headers present in mapping keys with non-null values can be forwarded
- This enforces stricter validation than the previous unsupported list approach
Configuration can be loaded from:
- Remote URL (default): Fetches from GitHub repository
- Local file: Set LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True to use bundled config only
Environment Variables:
- LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS: Set to "True" to disable remote fetching
- LITELLM_ANTHROPIC_BETA_HEADERS_URL: Custom URL for remote config (optional)
"""
import json
import os
from importlib.resources import files
from typing import Dict, List, Optional, Set
import httpx
from litellm.litellm_core_utils.litellm_logging import verbose_logger
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: Optional[Dict] = None
class GetAnthropicBetaHeadersConfig:
"""
Handles fetching, validating, and loading the Anthropic beta headers configuration.
Similar to GetModelCostMap, this class manages the lifecycle of the beta headers
configuration with support for remote fetching and local fallback.
"""
@staticmethod
def load_local_beta_headers_config() -> Dict:
"""Load the local backup beta headers config bundled with the package."""
try:
content = json.loads(
files("litellm")
.joinpath("anthropic_beta_headers_config.json")
.read_text(encoding="utf-8")
)
return content
except Exception as e:
verbose_logger.error(f"Failed to load local beta headers config: {e}")
# Return empty config as fallback
return {
"anthropic": {},
"azure_ai": {},
"bedrock": {},
"bedrock_converse": {},
"vertex_ai": {},
"provider_aliases": {}
}
@staticmethod
def _check_is_valid_dict(fetched_config: dict) -> bool:
"""Check if fetched config is a non-empty dict with expected structure."""
if not isinstance(fetched_config, dict):
verbose_logger.warning(
"LiteLLM: Fetched beta headers config is not a dict (type=%s). "
"Falling back to local backup.",
type(fetched_config).__name__,
)
return False
if len(fetched_config) == 0:
verbose_logger.warning(
"LiteLLM: Fetched beta headers config is empty. "
"Falling back to local backup.",
)
return False
# Check for at least one provider key
provider_keys = ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai"]
has_provider = any(key in fetched_config for key in provider_keys)
if not has_provider:
verbose_logger.warning(
"LiteLLM: Fetched beta headers config missing provider keys. "
"Falling back to local backup.",
)
return False
return True
@classmethod
def validate_beta_headers_config(cls, fetched_config: dict) -> bool:
"""
Validate the integrity of a fetched beta headers config.
Returns True if all checks pass, False otherwise.
"""
return cls._check_is_valid_dict(fetched_config)
@staticmethod
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> dict:
"""
Fetch the beta headers config from a remote URL.
Returns the parsed JSON dict. Raises on network/parse errors
(caller is expected to handle).
"""
response = httpx.get(url, timeout=timeout)
response.raise_for_status()
return response.json()
def get_beta_headers_config(url: str) -> dict:
"""
Public entry point — returns the beta headers config dict.
1. If ``LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS`` is set, uses the local backup only.
2. Otherwise fetches from ``url``, validates integrity, and falls back
to the local backup on any failure.
Args:
url: URL to fetch the remote beta headers configuration from
Returns:
Dict containing the beta headers configuration
"""
# Check if local-only mode is enabled
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
try:
content = GetAnthropicBetaHeadersConfig.fetch_remote_beta_headers_config(url)
except Exception as e:
verbose_logger.warning(
"LiteLLM: Failed to fetch remote beta headers config from %s: %s. "
"Falling back to local backup.",
url,
str(e),
)
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
# Validate the fetched config
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content):
verbose_logger.warning(
"LiteLLM: Fetched beta headers config failed integrity check. "
"Using local backup instead. url=%s",
url,
)
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
return content
def _load_beta_headers_config() -> Dict:
"""
Load the beta headers configuration from JSON file.
Uses caching to avoid repeated file reads.
Load the beta headers configuration.
Uses caching to avoid repeated fetches/file reads.
This function is called by all public API functions and manages the global cache.
Returns:
Dict containing the beta headers configuration
@ -36,26 +175,27 @@ def _load_beta_headers_config() -> Dict:
if _BETA_HEADERS_CONFIG is not None:
return _BETA_HEADERS_CONFIG
config_path = os.path.join(
os.path.dirname(__file__),
"anthropic_beta_headers_config.json"
)
# Get the URL from environment or use default
from litellm import anthropic_beta_headers_url
try:
with open(config_path, "r") as f:
_BETA_HEADERS_CONFIG = json.load(f)
verbose_logger.debug(f"Loaded beta headers config from {config_path}")
return _BETA_HEADERS_CONFIG
except Exception as e:
verbose_logger.error(f"Failed to load beta headers config: {e}")
# Return empty config as fallback (empty mappings)
return {
"anthropic": {},
"azure_ai": {},
"bedrock": {},
"bedrock_converse": {},
"vertex_ai": {}
}
_BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)
verbose_logger.debug("Loaded and cached beta headers config")
return _BETA_HEADERS_CONFIG
def reload_beta_headers_config() -> Dict:
"""
Force reload the beta headers configuration from source (remote or local).
Clears the cache and fetches fresh configuration.
Returns:
Dict containing the newly loaded beta headers configuration
"""
global _BETA_HEADERS_CONFIG
_BETA_HEADERS_CONFIG = None
verbose_logger.info("Reloading beta headers config (cache cleared)")
return _load_beta_headers_config()
def get_provider_name(provider: str) -> str:

64
litellm/batches/batch_utils.py
View file

@ -39,11 +39,19 @@ async def _handle_completed_batch(
batch: Batch,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
model_name: Optional[str] = None,
litellm_params: Optional[dict] = None,
) -> Tuple[float, Usage, List[str]]:
"""Helper function to process a completed batch and handle logging"""
"""Helper function to process a completed batch and handle logging
Args:
batch: The batch object
custom_llm_provider: The LLM provider
model_name: Optional model name
litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.)
"""
# Get batch results
file_content_dictionary = await _get_batch_output_file_content_as_dictionary(
batch, custom_llm_provider
batch, custom_llm_provider, litellm_params=litellm_params
)
# Calculate costs and usage
@ -187,9 +195,16 @@ def calculate_vertex_ai_batch_cost_and_usage(
async def _get_batch_output_file_content_as_dictionary(
batch: Batch,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
litellm_params: Optional[dict] = None,
) -> List[dict]:
"""
Get the batch output file content as a list of dictionaries
Args:
batch: The batch object
custom_llm_provider: The LLM provider
litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.)
Required for Azure and other providers that need authentication
"""
from litellm.files.main import afile_content
from litellm.proxy.openai_files_endpoints.common_utils import (
@ -211,13 +226,50 @@ async def _get_batch_output_file_content_as_dictionary(
except (IndexError, AttributeError) as e:
verbose_logger.error(f"Failed to extract LLM output file ID from unified file ID: {batch.output_file_id}, error: {e}")
_file_content = await afile_content(
file_id=file_id,
custom_llm_provider=custom_llm_provider,
)
# Build kwargs for afile_content with credentials from litellm_params
file_content_kwargs = {
"file_id": file_id,
"custom_llm_provider": custom_llm_provider,
}
# Extract and add credentials for file access
credentials = _extract_file_access_credentials(litellm_params)
file_content_kwargs.update(credentials)
_file_content = await afile_content(**file_content_kwargs)
return _get_file_content_as_dictionary(_file_content.content)
def _extract_file_access_credentials(litellm_params: Optional[dict]) -> dict:
"""
Extract credentials from litellm_params for file access operations.
This method extracts relevant authentication and configuration parameters
needed for accessing files across different providers (Azure, Vertex AI, etc.).
Args:
litellm_params: Dictionary containing litellm parameters with credentials
Returns:
Dictionary containing only the credentials needed for file access
"""
credentials = {}
if litellm_params:
# List of credential keys that should be passed to file operations
credential_keys = [
"api_key", "api_base", "api_version", "organization",
"azure_ad_token", "azure_ad_token_provider",
"vertex_project", "vertex_location", "vertex_credentials",
"timeout", "max_retries"
]
for key in credential_keys:
if key in litellm_params:
credentials[key] = litellm_params[key]
return credentials
def _get_file_content_as_dictionary(file_content: bytes) -> List[dict]:
"""
Get the file content as a list of dictionaries from JSON Lines format

66
litellm/caching/dual_cache.py
View file

@ -12,7 +12,8 @@ import asyncio
import time
import traceback
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING, Any, List, Optional, Union
from threading import Lock
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
if TYPE_CHECKING:
from litellm.types.caching import RedisPipelineIncrementOperation
@ -71,6 +72,7 @@ class DualCache(BaseCache):
self.last_redis_batch_access_time = LimitedSizeOrderedDict(
max_size=default_max_redis_batch_cache_size
)
self._last_redis_batch_access_time_lock = Lock()
self.redis_batch_cache_expiry = (
default_redis_batch_cache_expiry
or litellm.default_redis_batch_cache_expiry
@ -236,22 +238,46 @@ class DualCache(BaseCache):
except Exception:
verbose_logger.error(traceback.format_exc())
def get_redis_batch_keys(
def _reserve_redis_batch_keys(
self,
current_time: float,
keys: List[str],
result: List[Any],
) -> List[str]:
sublist_keys = []
for key, value in zip(keys, result):
if value is None:
) -> Tuple[List[str], Dict[str, Optional[float]]]:
"""
Atomically choose keys to fetch from Redis and reserve their access time.
This prevents check-then-act races under concurrent async callers.
"""
sublist_keys: List[str] = []
previous_access_times: Dict[str, Optional[float]] = {}
with self._last_redis_batch_access_time_lock:
for key, value in zip(keys, result):
if value is not None:
continue
if (
key not in self.last_redis_batch_access_time
or current_time - self.last_redis_batch_access_time[key]
>= self.redis_batch_cache_expiry
):
sublist_keys.append(key)
return sublist_keys
previous_access_times[key] = self.last_redis_batch_access_time.get(
key
)
self.last_redis_batch_access_time[key] = current_time
return sublist_keys, previous_access_times
def _rollback_redis_batch_key_reservations(
self, previous_access_times: Dict[str, Optional[float]]
) -> None:
with self._last_redis_batch_access_time_lock:
for key, previous_time in previous_access_times.items():
if previous_time is None:
self.last_redis_batch_access_time.pop(key, None)
else:
self.last_redis_batch_access_time[key] = previous_time
async def async_batch_get_cache(
self,
@ -276,19 +302,23 @@ class DualCache(BaseCache):
- check the redis cache
"""
current_time = time.time()
sublist_keys = self.get_redis_batch_keys(current_time, keys, result)
sublist_keys, previous_access_times = self._reserve_redis_batch_keys(
current_time, keys, result
)
# Only hit Redis if the last access time was more than 5 seconds ago
# Only hit Redis if enough time has passed since last access.
if len(sublist_keys) > 0:
# If not found in in-memory cache, try fetching from Redis
redis_result = await self.redis_cache.async_batch_get_cache(
sublist_keys, parent_otel_span=parent_otel_span
)
# Update the last access time for ALL queried keys
# This includes keys with None values to throttle repeated Redis queries
for key in sublist_keys:
self.last_redis_batch_access_time[key] = current_time
try:
# If not found in in-memory cache, try fetching from Redis
redis_result = await self.redis_cache.async_batch_get_cache(
sublist_keys, parent_otel_span=parent_otel_span
)
except Exception:
# Do not throttle subsequent callers if the Redis read fails.
self._rollback_redis_batch_key_reservations(
previous_access_times
)
raise
# Short-circuit if redis_result is None or contains only None values
if redis_result is None or all(v is None for v in redis_result.values()):

144
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -227,6 +227,84 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return input_items, instructions
def _map_optional_params_to_responses_api_request(
self,
optional_params: dict,
responses_api_request: "ResponsesAPIOptionalRequestParams",
) -> None:
"""Map optional_params into responses_api_request (mutates in place)."""
for key, value in optional_params.items():
if value is None:
continue
if key in ("max_tokens", "max_completion_tokens"):
responses_api_request["max_output_tokens"] = value
elif key == "tools" and value is not None:
responses_api_request["tools"] = (
self._convert_tools_to_responses_format(
cast(List[Dict[str, Any]], value)
)
)
elif key == "response_format":
text_format = self._transform_response_format_to_text_format(value)
if text_format:
responses_api_request["text"] = text_format # type: ignore
elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
responses_api_request[key] = value # type: ignore
elif key == "previous_response_id":
responses_api_request["previous_response_id"] = value
elif key == "reasoning_effort":
responses_api_request["reasoning"] = self._map_reasoning_effort(value)
elif key == "web_search_options":
self._add_web_search_tool(responses_api_request, value)
def _build_sanitized_litellm_params(
self, litellm_params: dict
) -> Dict[str, Any]:
"""Build sanitized litellm_params with merged metadata."""
responses_optional_param_keys = set(
ResponsesAPIOptionalRequestParams.__annotations__.keys()
)
sanitized: Dict[str, Any] = {
key: value
for key, value in litellm_params.items()
if key not in responses_optional_param_keys
}
legacy_metadata = litellm_params.get("metadata")
existing_litellm_metadata = litellm_params.get("litellm_metadata")
merged_litellm_metadata: Dict[str, Any] = {}
if isinstance(legacy_metadata, dict):
merged_litellm_metadata.update(legacy_metadata)
if isinstance(existing_litellm_metadata, dict):
merged_litellm_metadata.update(existing_litellm_metadata)
if merged_litellm_metadata:
sanitized["litellm_metadata"] = merged_litellm_metadata
else:
sanitized.pop("litellm_metadata", None)
return sanitized
def _merge_responses_api_request_into_request_data(
self,
request_data: Dict[str, Any],
responses_api_request: "ResponsesAPIOptionalRequestParams",
instructions: Optional[str],
) -> None:
"""Add non-None values from responses_api_request into request_data."""
for key, value in responses_api_request.items():
if value is None:
continue
if key == "instructions" and instructions:
request_data["instructions"] = instructions
elif key == "stream_options" and isinstance(value, dict):
request_data["stream_options"] = value.get("include_obfuscation")
elif key == "user" and isinstance(value, str):
# OpenAI API requires user param to be max 64 chars - truncate if longer
if len(value) <= 64:
request_data["user"] = value
else:
request_data["user"] = value[:64]
else:
request_data[key] = value
def transform_request(
self,
model: str,
@ -251,34 +329,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if instructions:
responses_api_request["instructions"] = instructions
# Map optional parameters
for key, value in optional_params.items():
if value is None:
continue
if key in ("max_tokens", "max_completion_tokens"):
responses_api_request["max_output_tokens"] = value
elif key == "tools" and value is not None:
# Convert chat completion tools to responses API tools format
responses_api_request["tools"] = (
self._convert_tools_to_responses_format(
cast(List[Dict[str, Any]], value)
)
)
elif key == "response_format":
# Convert response_format to text.format
text_format = self._transform_response_format_to_text_format(value)
if text_format:
responses_api_request["text"] = text_format # type: ignore
elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
responses_api_request[key] = value # type: ignore
elif key == "previous_response_id":
responses_api_request["previous_response_id"] = value
elif key == "reasoning_effort":
responses_api_request["reasoning"] = self._map_reasoning_effort(value)
elif key == "web_search_options":
self._add_web_search_tool(responses_api_request, value)
self._map_optional_params_to_responses_api_request(
optional_params, responses_api_request
)
# Get stream parameter from litellm_params if not in optional_params
stream = optional_params.get("stream") or litellm_params.get("stream", False)
verbose_logger.debug(f"Chat provider: Stream parameter: {stream}")
@ -302,26 +356,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
setattr(litellm_logging_obj, "call_type", CallTypes.responses.value)
responses_optional_param_keys = set(
ResponsesAPIOptionalRequestParams.__annotations__.keys()
sanitized_litellm_params = self._build_sanitized_litellm_params(
litellm_params
)
sanitized_litellm_params: Dict[str, Any] = {
key: value
for key, value in litellm_params.items()
if key not in responses_optional_param_keys
}
legacy_metadata = litellm_params.get("metadata")
existing_litellm_metadata = litellm_params.get("litellm_metadata")
merged_litellm_metadata: Dict[str, Any] = {}
if isinstance(legacy_metadata, dict):
merged_litellm_metadata.update(legacy_metadata)
if isinstance(existing_litellm_metadata, dict):
merged_litellm_metadata.update(existing_litellm_metadata)
if merged_litellm_metadata:
sanitized_litellm_params["litellm_metadata"] = merged_litellm_metadata
else:
sanitized_litellm_params.pop("litellm_metadata", None)
request_data = {
"model": api_model,
@ -335,18 +372,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
f"Chat provider: Final request model={api_model}, input_items={len(input_items)}"
)
# Add non-None values from responses_api_request
for key, value in responses_api_request.items():
if value is not None:
if key == "instructions" and instructions:
request_data["instructions"] = instructions
elif key == "stream_options" and isinstance(value, dict):
request_data["stream_options"] = value.get("include_obfuscation")
elif key == "user": # string can't be longer than 64 characters
if isinstance(value, str) and len(value) <= 64:
request_data["user"] = value
else:
request_data[key] = value
self._merge_responses_api_request_into_request_data(
request_data, responses_api_request, instructions
)
if headers:
request_data["extra_headers"] = headers

70
litellm/constants.py
View file

@ -101,9 +101,12 @@ MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE = int(
MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL = int(
os.getenv("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", "3600")
)
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL = int(
os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10")
)
# Default npm cache directory for STDIO MCP servers.
# npm/npx needs a writable cache dir; in containers the default (~/.npm)
# may not exist or be read-only. /tmp is always writable.
MCP_NPM_CACHE_DIR = os.getenv("MCP_NPM_CACHE_DIR", "/tmp/.npm_mcp_cache")
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10"))
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
@ -126,7 +129,7 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int(
# Maximum number of callbacks that can be registered
# This prevents callbacks from exponentially growing and consuming CPU resources
# Override with LITELLM_MAX_CALLBACKS env var for large deployments (e.g., many teams with guardrails)
MAX_CALLBACKS = get_env_int("LITELLM_MAX_CALLBACKS", 30)
MAX_CALLBACKS = get_env_int("LITELLM_MAX_CALLBACKS", 100)
# Generic fallback for unknown models
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int(
@ -162,15 +165,19 @@ _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client fo
# Aiohttp connection pooling - prevents memory leaks from unbounded connection growth
# Set to 0 for unlimited (not recommended for production)
AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 300))
AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 50))
AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int(
os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 50)
)
AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120))
AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300))
# enable_cleanup_closed is only needed for Python versions with the SSL leak bug
# Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960)
# Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78
AIOHTTP_NEEDS_CLEANUP_CLOSED = (
(3, 13, 0) <= sys.version_info < (3, 13, 1) or sys.version_info < (3, 12, 7)
)
AIOHTTP_NEEDS_CLEANUP_CLOSED = (3, 13, 0) <= sys.version_info < (
3,
13,
1,
) or sys.version_info < (3, 12, 7)
# WebSocket constants
# Default to None (unlimited) to match OpenAI's official agents SDK behavior
@ -208,15 +215,15 @@ REDIS_UPDATE_BUFFER_KEY = "litellm_spend_update_buffer"
REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_spend_update_buffer"
REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_team_spend_update_buffer"
REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_org_spend_update_buffer"
REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_end_user_spend_update_buffer"
REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY = (
"litellm_daily_end_user_spend_update_buffer"
)
REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_agent_spend_update_buffer"
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_tag_spend_update_buffer"
MAX_REDIS_BUFFER_DEQUEUE_COUNT = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100))
MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 2000))
# Bounds asyncio.Queue() instances (log queues, spend update queues, etc.) to prevent unbounded memory growth
LITELLM_ASYNCIO_QUEUE_MAXSIZE = int(
os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000)
)
LITELLM_ASYNCIO_QUEUE_MAXSIZE = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000))
MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int(
os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000)
)
@ -338,7 +345,9 @@ MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
#### Networking settings ####
request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds
DEFAULT_A2A_AGENT_TIMEOUT: float = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)) # 10 minutes
DEFAULT_A2A_AGENT_TIMEOUT: float = float(
os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)
) # 10 minutes
# Patterns that indicate a localhost/internal URL in A2A agent cards that should be
# replaced with the original base_url. This is a common misconfiguration where
# developers deploy agents with development URLs in their agent cards.
@ -390,8 +399,12 @@ DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE = os.getenv(
"DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE", "streaming.chunk.yield"
)
EMAIL_BUDGET_ALERT_TTL = int(os.getenv("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60)) # 24 hours in seconds
EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE = float(os.getenv("EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8)) # 80% of max budget
EMAIL_BUDGET_ALERT_TTL = int(
os.getenv("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60)
) # 24 hours in seconds
EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE = float(
os.getenv("EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8)
) # 80% of max budget
############### LLM Provider Constants ###############
### ANTHROPIC CONSTANTS ###
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION = os.getenv(
@ -1011,10 +1024,12 @@ BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[
BEDROCK_CONVERSE_MODELS = [
"qwen.qwen3-coder-480b-a35b-v1:0",
"qwen.qwen3-coder-next",
"qwen.qwen3-235b-a22b-2507-v1:0",
"qwen.qwen3-coder-30b-a3b-v1:0",
"qwen.qwen3-32b-v1:0",
"deepseek.v3-v1:0",
"deepseek.v3.2",
"openai.gpt-oss-20b-1:0",
"openai.gpt-oss-120b-1:0",
"anthropic.claude-haiku-4-5-20251001-v1:0",
@ -1057,6 +1072,8 @@ BEDROCK_CONVERSE_MODELS = [
"amazon.nova-pro-v1:0",
"writer.palmyra-x4-v1:0",
"writer.palmyra-x5-v1:0",
"minimax.minimax-m2.1",
"moonshotai.kimi-k2.5",
]
@ -1141,7 +1158,17 @@ known_tokenizer_config = {
}
OPENAI_FINISH_REASONS = ["stop", "length", "function_call", "content_filter", "null", "finish_reason_unspecified", "malformed_function_call", "guardrail_intervened", "eos"]
OPENAI_FINISH_REASONS = [
"stop",
"length",
"function_call",
"content_filter",
"null",
"finish_reason_unspecified",
"malformed_function_call",
"guardrail_intervened",
"eos",
]
HUMANLOOP_PROMPT_CACHE_TTL_SECONDS = int(
os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60)
) # 1 minute
@ -1241,8 +1268,8 @@ CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session"
CLI_JWT_TOKEN_NAME = "cli-jwt-token"
# Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility
CLI_JWT_EXPIRATION_HOURS = int(
os.getenv("CLI_JWT_EXPIRATION_HOURS")
or os.getenv("LITELLM_CLI_JWT_EXPIRATION_HOURS")
os.getenv("CLI_JWT_EXPIRATION_HOURS")
or os.getenv("LITELLM_CLI_JWT_EXPIRATION_HOURS")
or 24
)
@ -1333,6 +1360,9 @@ SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"]
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(
os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)
)
DEFAULT_ACCESS_GROUP_CACHE_TTL = int(
os.getenv("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600)
)
# Sentry Scrubbing Configuration
SENTRY_DENYLIST = [
@ -1423,9 +1453,7 @@ MICROSOFT_USER_EMAIL_ATTRIBUTE = str(
MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE", "displayName")
)
MICROSOFT_USER_ID_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_ID_ATTRIBUTE", "id")
)
MICROSOFT_USER_ID_ATTRIBUTE = str(os.getenv("MICROSOFT_USER_ID_ATTRIBUTE", "id"))
MICROSOFT_USER_FIRST_NAME_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_FIRST_NAME_ATTRIBUTE", "givenName")
)

28
litellm/cost_calculator.py
View file

@ -74,6 +74,7 @@ from litellm.llms.vertex_ai.cost_calculator import (
from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_router
from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
ImageGenerationRequestQuality,
@ -150,32 +151,33 @@ def _get_additional_costs(
) -> Optional[dict]:
"""
Calculate additional costs beyond standard token costs.
This function delegates to provider-specific config classes to calculate
any additional costs like routing fees, infrastructure costs, etc.
Args:
model: The model name
custom_llm_provider: The provider name (optional)
prompt_tokens: Number of prompt tokens
completion_tokens: Number of completion tokens
Returns:
Optional dictionary with cost names and amounts, or None if no additional costs
"""
if not custom_llm_provider:
return None
try:
config_class = None
if custom_llm_provider == "azure_ai":
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
config_class = AzureFoundryModelInfo.get_azure_ai_config_for_model(model)
# Add more providers here as needed
# elif custom_llm_provider == "other_provider":
# config_class = get_other_provider_config(model)
if config_class and hasattr(config_class, 'calculate_additional_costs'):
if config_class and hasattr(config_class, "calculate_additional_costs"):
return config_class.calculate_additional_costs(
model=model,
prompt_tokens=prompt_tokens,
@ -183,7 +185,7 @@ def _get_additional_costs(
)
except Exception as e:
verbose_logger.debug(f"Error calculating additional costs: {e}")
return None
@ -748,6 +750,8 @@ def _infer_call_type(
return "image_generation"
elif isinstance(completion_response, TextCompletionResponse):
return "text_completion"
elif isinstance(completion_response, LiteLLMSendMessageResponse):
return "send_message"
return call_type
@ -1037,9 +1041,9 @@ def completion_cost( # noqa: PLR0915
or isinstance(completion_response, dict)
): # tts returns a custom class
if isinstance(completion_response, dict):
usage_obj: Optional[
Union[dict, Usage]
] = completion_response.get("usage", {})
usage_obj: Optional[Union[dict, Usage]] = (
completion_response.get("usage", {})
)
else:
usage_obj = getattr(completion_response, "usage", {})
if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects(
@ -1393,7 +1397,7 @@ def completion_cost( # noqa: PLR0915
service_tier=service_tier,
response=completion_response,
)
# Get additional costs from provider (e.g., routing fees, infrastructure costs)
additional_costs = _get_additional_costs(
model=model,
@ -1401,7 +1405,7 @@ def completion_cost( # noqa: PLR0915
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
)
_final_cost = (
prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar
)

2
litellm/integrations/cloudzero/transform.py
View file

@ -141,7 +141,7 @@ class CBFTransformer:
# Required CBF fields
'time/usage_start': usage_date.isoformat() if usage_date else None, # Required: ISO-formatted UTC datetime
'cost/cost': float(row.get('spend', 0.0)), # Required: billed cost
'resource/id': model, # Send model name
'resource/id': resource_id, # CZRN (CloudZero Resource Name)
# Usage metrics for token consumption
'usage/amount': total_tokens, # Numeric value of tokens consumed

37
litellm/integrations/custom_guardrail.py
View file

@ -30,6 +30,11 @@ from litellm.types.utils import (
StandardLoggingGuardrailInformation,
)
try:
from fastapi.exceptions import HTTPException
except ImportError:
HTTPException = None # type: ignore
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
dc = DualCache()
@ -624,7 +629,9 @@ class CustomGuardrail(CustomLogger):
This gets logged on downsteam Langfuse, DataDog, etc.
"""
# Convert None to empty dict to satisfy type requirements
guardrail_response = {} if response is None else response
guardrail_response: Union[Dict[str, Any], str] = (
{} if response is None else response
)
# For apply_guardrail functions in custom_code_guardrail scenario,
# simplify the logged response to "allow", "deny", or "mask"
@ -648,6 +655,27 @@ class CustomGuardrail(CustomLogger):
)
return response
@staticmethod
def _is_guardrail_intervention(e: Exception) -> bool:
"""
Returns True if the exception represents an intentional guardrail block
(this was logged previously as an API failure - guardrail_failed_to_respond).
Guardrails signal intentional blocks by raising:
- HTTPException with status 400 (content policy violation)
- ModifyResponseException (passthrough mode violation)
"""
if isinstance(e, ModifyResponseException):
return True
if (
HTTPException is not None
and isinstance(e, HTTPException)
and e.status_code == 400
):
return True
return False
def _process_error(
self,
e: Exception,
@ -662,6 +690,11 @@ class CustomGuardrail(CustomLogger):
This gets logged on downsteam Langfuse, DataDog, etc.
"""
guardrail_status: GuardrailStatus = (
"guardrail_intervened"
if self._is_guardrail_intervention(e)
else "guardrail_failed_to_respond"
)
# For custom_code_guardrail scenario, log as "deny" instead of full exception
# Check if this is from custom_code_guardrail by checking the class name
guardrail_response: Union[Exception, str] = e
@ -671,7 +704,7 @@ class CustomGuardrail(CustomLogger):
self.add_standard_logging_guardrail_information_to_request_data(
guardrail_json_response=guardrail_response,
request_data=request_data,
guardrail_status="guardrail_failed_to_respond",
guardrail_status=guardrail_status,
duration=duration,
start_time=start_time,
end_time=end_time,

98
litellm/integrations/custom_logger.py
View file

@ -774,15 +774,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
self, model_call_details: Dict
) -> Dict:
"""
Only redacts messages and responses when self.turn_off_message_logging is True
Redacts or excludes fields from StandardLoggingPayload before callbacks receive it.
This method handles two features:
1. turn_off_message_logging: When True, redacts messages and responses
2. standard_logging_payload_excluded_fields: Removes specified fields entirely
By default, self.turn_off_message_logging is False and this does nothing.
Return a redacted deepcopy of the provided logging payload.
Return a modified copy of the provided logging payload.
This is useful for logging payloads that contain sensitive information.
"""
import litellm
from copy import copy
from litellm import Choices, Message, ModelResponse
@ -790,14 +792,17 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
turn_off_message_logging: bool = getattr(
self, "turn_off_message_logging", False
)
excluded_fields: Optional[List[str]] = getattr(
litellm, "standard_logging_payload_excluded_fields", None
)
if turn_off_message_logging is False:
# Early return if no processing needed
if turn_off_message_logging is False and not excluded_fields:
return model_call_details
# Only make a shallow copy of the top-level dict to avoid deepcopy issues
# with complex objects like AuthenticationError that may be present
model_call_details_copy = copy(model_call_details)
redacted_str = "redacted-by-litellm"
standard_logging_object = model_call_details.get("standard_logging_object")
if standard_logging_object is None:
return model_call_details_copy
@ -805,39 +810,58 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
# Make a copy of just the standard_logging_object to avoid modifying the original
standard_logging_object_copy = copy(standard_logging_object)
if standard_logging_object_copy.get("messages") is not None:
standard_logging_object_copy["messages"] = [
Message(content=redacted_str).model_dump()
]
# Handle excluded fields - remove them entirely from the payload
if excluded_fields:
for field in excluded_fields:
if field in standard_logging_object_copy:
del standard_logging_object_copy[field]
if standard_logging_object_copy.get("response") is not None:
response = standard_logging_object_copy["response"]
# Check if this is a ResponsesAPIResponse (has "output" field)
if isinstance(response, dict) and "output" in response:
# Make a copy to avoid modifying the original
from copy import deepcopy
# Handle turn_off_message_logging - redact messages and responses (if not already excluded)
if turn_off_message_logging:
redacted_str = "redacted-by-litellm"
response_copy = deepcopy(response)
# Redact content in output array
if isinstance(response_copy.get("output"), list):
for output_item in response_copy["output"]:
if isinstance(output_item, dict) and "content" in output_item:
if isinstance(output_item["content"], list):
# Redact text in content items
for content_item in output_item["content"]:
if (
isinstance(content_item, dict)
and "text" in content_item
):
content_item["text"] = redacted_str
standard_logging_object_copy["response"] = response_copy
else:
# Standard ModelResponse format
model_response = ModelResponse(
choices=[Choices(message=Message(content=redacted_str))]
)
model_response_dict = model_response.model_dump()
standard_logging_object_copy["response"] = model_response_dict
if (
"messages" not in (excluded_fields or [])
and standard_logging_object_copy.get("messages") is not None
):
standard_logging_object_copy["messages"] = [
Message(content=redacted_str).model_dump()
]
if (
"response" not in (excluded_fields or [])
and standard_logging_object_copy.get("response") is not None
):
response = standard_logging_object_copy["response"]
# Check if this is a ResponsesAPIResponse (has "output" field)
if isinstance(response, dict) and "output" in response:
# Make a copy to avoid modifying the original
from copy import deepcopy
response_copy = deepcopy(response)
# Redact content in output array
if isinstance(response_copy.get("output"), list):
for output_item in response_copy["output"]:
if (
isinstance(output_item, dict)
and "content" in output_item
):
if isinstance(output_item["content"], list):
# Redact text in content items
for content_item in output_item["content"]:
if (
isinstance(content_item, dict)
and "text" in content_item
):
content_item["text"] = redacted_str
standard_logging_object_copy["response"] = response_copy
else:
# Standard ModelResponse format
model_response = ModelResponse(
choices=[Choices(message=Message(content=redacted_str))]
)
model_response_dict = model_response.model_dump()
standard_logging_object_copy["response"] = model_response_dict
model_call_details_copy["standard_logging_object"] = (
standard_logging_object_copy

17
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -1,6 +1,7 @@
import base64
import json # <--- NEW
import os
from datetime import datetime
from typing import TYPE_CHECKING, Any, Optional, Union
from litellm._logging import verbose_logger
@ -392,6 +393,22 @@ class LangfuseOtelLogger(OpenTelemetry):
return dynamic_headers
def create_litellm_proxy_request_started_span(
self,
start_time: datetime,
headers: dict,
) -> Optional[Span]:
"""
Override to prevent creating empty proxy request spans.
Langfuse should only receive spans for actual LLM calls, not for
internal proxy operations (auth, postgres, proxy_pre_call, etc.).
By returning None, we prevent the parent span from being created,
which in turn prevents empty traces from being sent to Langfuse.
"""
return None
async def async_service_success_hook(self, *args, **kwargs):
"""
Langfuse should not receive service success logs.

33
litellm/litellm_core_utils/api_route_to_call_types.py
View file

@ -3,6 +3,9 @@ Dictionary mapping API routes to their corresponding CallTypes in LiteLLM.
This dictionary maps each API endpoint to the CallTypes that can be used for that route.
Each route can have both async (prefixed with 'a') and sync call types.
Route patterns may contain placeholders like {agent_id}, {model}, {batch_id}; these
match a single path segment when resolving call types for a concrete path.
"""
from typing import List, Optional
@ -10,17 +13,43 @@ from typing import List, Optional
from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes
def _route_matches_pattern(route: str, pattern: str) -> bool:
"""
Return True if the concrete route matches the pattern.
Pattern segments like {param} match any single path segment.
"""
route_parts = route.strip("/").split("/")
pattern_parts = pattern.strip("/").split("/")
if len(route_parts) != len(pattern_parts):
return False
for r, p in zip(route_parts, pattern_parts):
if p.startswith("{") and p.endswith("}"):
continue
if r != p:
return False
return True
def get_call_types_for_route(route: str) -> Optional[List[CallTypes]]:
"""
Get the list of CallTypes for a given API route.
Supports both exact keys and dynamic patterns (e.g. /a2a/my-agent/message/send
matches /a2a/{agent_id}/message/send).
Args:
route: API route path (e.g., "/chat/completions")
route: API route path (e.g., "/chat/completions" or "/a2a/my-pydantic-agent/message/send")
Returns:
List of CallTypes for that route, or None if route not found
"""
return API_ROUTE_TO_CALL_TYPES.get(route, None)
exact = API_ROUTE_TO_CALL_TYPES.get(route, None)
if exact is not None:
return exact
for pattern, call_types in API_ROUTE_TO_CALL_TYPES.items():
if _route_matches_pattern(route, pattern):
return call_types
return None
def get_routes_for_call_type(call_type: CallTypes) -> list:

24
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -70,6 +70,11 @@ class ExceptionCheckers:
Check if an error string indicates a context window exceeded error.
"""
_error_str_lowercase = error_str.lower()
# Exclude param validation errors (e.g. OpenAI "user" param max 64 chars)
if "string_above_max_length" in _error_str_lowercase:
return False
if "invalid 'user'" in _error_str_lowercase and "string too long" in _error_str_lowercase:
return False
known_exception_substrings = [
"exceed context limit",
"this model's maximum context length is",
@ -98,16 +103,18 @@ class ExceptionCheckers:
"""
Check if an error string indicates a content policy violation error.
"""
_lower = error_str.lower()
known_exception_substrings = [
"invalid_request_error",
"content_policy_violation",
"responsibleaipolicyviolation",
"the response was filtered due to the prompt triggering azure openai's content management",
"your task failed as a result of our safety system",
"the model produced invalid content",
"content_filter_policy",
"your request was rejected as a result of our safety system",
]
for substring in known_exception_substrings:
if substring in error_str.lower():
if substring in _lower:
return True
return False
@ -2060,6 +2067,19 @@ def exception_type( # type: ignore # noqa: PLR0915
if isinstance(body_dict, dict):
if isinstance(body_dict.get("error"), dict):
azure_error_code = body_dict["error"].get("code") # type: ignore[index]
# Also check inner_error for
# ResponsibleAIPolicyViolation which indicates a
# content policy violation even when the top-level
# code is generic (e.g. "invalid_request_error").
if azure_error_code != "content_policy_violation":
_inner = (
body_dict["error"].get("inner_error") # type: ignore[index]
or body_dict["error"].get("innererror") # type: ignore[index]
)
if isinstance(_inner, dict) and _inner.get(
"code"
) == "ResponsibleAIPolicyViolation":
azure_error_code = "content_policy_violation"
else:
azure_error_code = body_dict.get("code")
except Exception:

5
litellm/litellm_core_utils/litellm_logging.py
View file

@ -2331,7 +2331,7 @@ class Logging(LiteLLMLoggingBaseClass):
result, LiteLLMBatch
):
litellm_params = self.litellm_params or {}
litellm_metadata = litellm_params.get("litellm_metadata", {})
litellm_metadata = litellm_params.get("litellm_metadata") or {}
if (
litellm_metadata.get("batch_ignore_default_logging", False) is True
): # polling job will query these frequently, don't spam db logs
@ -2369,6 +2369,7 @@ class Logging(LiteLLMLoggingBaseClass):
) = await _handle_completed_batch(
batch=result,
custom_llm_provider=self.custom_llm_provider,
litellm_params=self.litellm_params,
)
result._hidden_params["response_cost"] = response_cost
@ -3127,7 +3128,7 @@ class Logging(LiteLLMLoggingBaseClass):
self, dynamic_success_callbacks: Optional[List], global_callbacks: List
) -> List:
if dynamic_success_callbacks is None:
return global_callbacks
return list(global_callbacks)
return list(set(dynamic_success_callbacks + global_callbacks))
def _remove_internal_litellm_callbacks(self, callbacks: List) -> List:

9
litellm/litellm_core_utils/safe_json_dumps.py
View file

@ -1,6 +1,8 @@
import json
from typing import Any, Union
from pydantic import BaseModel
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
@ -41,6 +43,11 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str:
result = sorted([_serialize(item, seen, depth + 1) for item in obj])
seen.remove(id(obj))
return result
elif isinstance(obj, BaseModel):
dumped = obj.model_dump()
result = _serialize(dumped, seen, depth + 1)
seen.remove(id(obj))
return result
else:
# Fall back to string conversion for non-serializable objects.
try:
@ -49,4 +56,4 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str:
return "Unserializable Object"
safe_data = _serialize(data, set(), 0)
return json.dumps(safe_data, default=str)
return json.dumps(safe_data, default=str)

113
litellm/llms/a2a/chat/guardrail_translation/handler.py
View file

@ -10,6 +10,7 @@ A2A Protocol Format:
- Output: JSON-RPC 2.0 with result containing message/artifact parts
"""
import json
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from litellm._logging import verbose_proxy_logger
@ -206,6 +207,118 @@ class A2AGuardrailHandler(BaseTranslation):
response["result"] = result
return response
async def process_output_streaming_response(
self,
responses_so_far: List[Any],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
) -> List[Any]:
"""
Process A2A streaming output by applying guardrails to accumulated text.
responses_so_far can be a list of JSON-RPC 2.0 objects (dict or NDJSON str), e.g.:
- task with history, status-update, artifact-update (with result.artifact.parts),
- then status-update (final). Text is extracted from result.artifact.parts,
result.message.parts, result.parts, etc., concatenated in order, guardrailed once,
then the combined guardrailed text is written into the first chunk that had text
and all other text parts in other chunks are cleared (in-place).
"""
from litellm.llms.a2a.common_utils import extract_text_from_a2a_response
# Parse each item; keep alignment with responses_so_far (None where unparseable)
parsed: List[Optional[Dict[str, Any]]] = [None] * len(responses_so_far)
for i, item in enumerate(responses_so_far):
if isinstance(item, dict):
obj = item
elif isinstance(item, str):
try:
obj = json.loads(item.strip())
except (json.JSONDecodeError, TypeError):
continue
else:
continue
if isinstance(obj.get("result"), dict):
parsed[i] = obj
valid_parsed = [(i, obj) for i, obj in enumerate(parsed) if obj is not None]
if not valid_parsed:
return responses_so_far
# Collect text from each chunk in order (by original index in responses_so_far)
text_parts: List[str] = []
chunk_indices_with_text: List[int] = [] # indices into valid_parsed
for idx, (orig_i, obj) in enumerate(valid_parsed):
t = extract_text_from_a2a_response(obj)
if t:
text_parts.append(t)
chunk_indices_with_text.append(orig_i)
combined_text = "".join(text_parts)
if not combined_text:
return responses_so_far
request_data: dict = {"responses_so_far": responses_so_far}
user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
if user_metadata:
request_data["litellm_metadata"] = user_metadata
inputs = GenericGuardrailAPIInputs(texts=[combined_text])
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=request_data,
input_type="response",
logging_obj=litellm_logging_obj,
)
guardrailed_texts = guardrailed_inputs.get("texts", [])
if not guardrailed_texts:
return responses_so_far
guardrailed_text = guardrailed_texts[0]
# Find first chunk (by original index) that has text; put full guardrailed text there and clear rest
first_chunk_with_text: Optional[int] = (
chunk_indices_with_text[0] if chunk_indices_with_text else None
)
for orig_i, obj in valid_parsed:
result = obj.get("result", {})
if not isinstance(result, dict):
continue
texts_in_chunk: List[str] = []
mappings: List[Tuple[Tuple[str, ...], int]] = []
self._extract_texts_from_result(
result=result,
texts_to_check=texts_in_chunk,
task_mappings=mappings,
)
if not mappings:
continue
if orig_i == first_chunk_with_text:
# Put full guardrailed text in first text part; clear others
for task_idx, (path, part_idx) in enumerate(mappings):
text = guardrailed_text if task_idx == 0 else ""
self._apply_text_to_path(
result=result,
path=path,
part_idx=part_idx,
text=text,
)
else:
for path, part_idx in mappings:
self._apply_text_to_path(
result=result,
path=path,
part_idx=part_idx,
text="",
)
# Write back to responses_so_far where we had NDJSON strings
for i, item in enumerate(responses_so_far):
if isinstance(item, str) and parsed[i] is not None:
responses_so_far[i] = json.dumps(parsed[i]) + "\n"
return responses_so_far
def _extract_texts_from_result(
self,
result: Dict[str, Any],

105
litellm/llms/anthropic/chat/transformation.py
View file

@ -208,29 +208,73 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
Filter out unsupported fields from JSON schema for Anthropic's output_format API.
Anthropic's output_format doesn't support certain JSON schema properties:
- maxItems: Not supported for array types
- minItems: Not supported for array types
- maxItems/minItems: Not supported for array types
- minimum/maximum: Not supported for numeric types
- minLength/maxLength: Not supported for string types
This function recursively removes these unsupported fields while preserving
all other valid schema properties.
This mirrors the transformation done by the Anthropic Python SDK.
See: https://platform.claude.com/docs/en/build-with-claude/structured-outputs#how-sdk-transformation-works
The SDK approach:
1. Remove unsupported constraints from schema
2. Add constraint info to description (e.g., "Must be at least 100")
3. Validate responses against original schema
Args:
schema: The JSON schema dictionary to filter
Returns:
A new dictionary with unsupported fields removed
A new dictionary with unsupported fields removed and descriptions updated
Related issue: https://github.com/BerriAI/litellm/issues/19444
Related issues:
- https://github.com/BerriAI/litellm/issues/19444
"""
if not isinstance(schema, dict):
return schema
unsupported_fields = {"maxItems", "minItems"}
# All numeric/string/array constraints not supported by Anthropic
unsupported_fields = {
"maxItems", "minItems", # array constraints
"minimum", "maximum", # numeric constraints
"exclusiveMinimum", "exclusiveMaximum", # numeric constraints
"minLength", "maxLength", # string constraints
}
# Build description additions from removed constraints
constraint_descriptions: list = []
constraint_labels = {
"minItems": "minimum number of items: {}",
"maxItems": "maximum number of items: {}",
"minimum": "minimum value: {}",
"maximum": "maximum value: {}",
"exclusiveMinimum": "exclusive minimum value: {}",
"exclusiveMaximum": "exclusive maximum value: {}",
"minLength": "minimum length: {}",
"maxLength": "maximum length: {}",
}
for field in unsupported_fields:
if field in schema:
constraint_descriptions.append(
constraint_labels[field].format(schema[field])
)
result: Dict[str, Any] = {}
# Update description with removed constraint info
if constraint_descriptions:
existing_desc = schema.get("description", "")
constraint_note = "Note: " + ", ".join(constraint_descriptions) + "."
if existing_desc:
result["description"] = existing_desc + " " + constraint_note
else:
result["description"] = constraint_note
for key, value in schema.items():
if key in unsupported_fields:
continue
if key == "description" and "description" in result:
# Already handled above
continue
if key == "properties" and isinstance(value, dict):
result[key] = {
@ -664,35 +708,34 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
reasoning_effort: Optional[Union[REASONING_EFFORT, str]],
model: str,
) -> Optional[AnthropicThinkingParam]:
if reasoning_effort is None or reasoning_effort == "none":
return None
if AnthropicConfig._is_claude_opus_4_6(model):
return AnthropicThinkingParam(
type="adaptive",
)
elif reasoning_effort == "low":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
)
elif reasoning_effort == "medium":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
)
elif reasoning_effort == "high":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
)
elif reasoning_effort == "minimal":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET,
)
else:
if reasoning_effort is None:
return None
elif reasoning_effort == "low":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
)
elif reasoning_effort == "medium":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
)
elif reasoning_effort == "high":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
)
elif reasoning_effort == "minimal":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET,
)
else:
raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}")
raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}")
def _extract_json_schema_from_response_format(
self, value: Optional[dict]

137
litellm/llms/anthropic/common_utils.py
View file

@ -38,9 +38,18 @@ def optionally_handle_anthropic_oauth(
Returns:
Tuple of (updated headers, api_key)
"""
# Check Authorization header (passthrough / forwarded requests)
auth_header = headers.get("authorization", "")
if auth_header and auth_header.startswith(f"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}"):
api_key = auth_header.replace("Bearer ", "")
headers.pop("x-api-key", None)
headers["anthropic-beta"] = ANTHROPIC_OAUTH_BETA_HEADER
headers["anthropic-dangerous-direct-browser-access"] = "true"
return headers, api_key
# Check api_key directly (standard chat/completion flow)
if api_key and api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX):
headers.pop("x-api-key", None)
headers["authorization"] = f"Bearer {api_key}"
headers["anthropic-beta"] = ANTHROPIC_OAUTH_BETA_HEADER
headers["anthropic-dangerous-direct-browser-access"] = "true"
return headers, api_key
@ -108,7 +117,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if tools is None:
return False
for tool in tools:
if "type" in tool and tool["type"].startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value):
if "type" in tool and tool["type"].startswith(
ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value
):
return True
return False
@ -134,111 +145,126 @@ class AnthropicModelInfo(BaseLLMModelInfo):
"""
if not tools:
return False
for tool in tools:
tool_type = tool.get("type", "")
if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]:
if tool_type in [
"tool_search_tool_regex_20251119",
"tool_search_tool_bm25_20251119",
]:
return True
return False
def is_programmatic_tool_calling_used(self, tools: Optional[List]) -> bool:
"""
Check if programmatic tool calling is being used (tools with allowed_callers field).
Returns True if any tool has allowed_callers containing 'code_execution_20250825'.
"""
if not tools:
return False
for tool in tools:
# Check top-level allowed_callers
allowed_callers = tool.get("allowed_callers", None)
if allowed_callers and isinstance(allowed_callers, list):
if "code_execution_20250825" in allowed_callers:
return True
# Check function.allowed_callers for OpenAI format tools
function = tool.get("function", {})
if isinstance(function, dict):
function_allowed_callers = function.get("allowed_callers", None)
if function_allowed_callers and isinstance(function_allowed_callers, list):
if function_allowed_callers and isinstance(
function_allowed_callers, list
):
if "code_execution_20250825" in function_allowed_callers:
return True
return False
def is_input_examples_used(self, tools: Optional[List]) -> bool:
"""
Check if input_examples is being used in any tools.
Returns True if any tool has input_examples field.
"""
if not tools:
return False
for tool in tools:
# Check top-level input_examples
input_examples = tool.get("input_examples", None)
if input_examples and isinstance(input_examples, list) and len(input_examples) > 0:
if (
input_examples
and isinstance(input_examples, list)
and len(input_examples) > 0
):
return True
# Check function.input_examples for OpenAI format tools
function = tool.get("function", {})
if isinstance(function, dict):
function_input_examples = function.get("input_examples", None)
if function_input_examples and isinstance(function_input_examples, list) and len(function_input_examples) > 0:
if (
function_input_examples
and isinstance(function_input_examples, list)
and len(function_input_examples) > 0
):
return True
return False
def is_effort_used(self, optional_params: Optional[dict], model: Optional[str] = None) -> bool:
def is_effort_used(
self, optional_params: Optional[dict], model: Optional[str] = None
) -> bool:
"""
Check if effort parameter is being used.
Returns True if effort-related parameters are present.
"""
if not optional_params:
return False
# Check if reasoning_effort is provided for Claude Opus 4.5
if model and ("opus-4-5" in model.lower() or "opus_4_5" in model.lower()):
reasoning_effort = optional_params.get("reasoning_effort")
if reasoning_effort and isinstance(reasoning_effort, str):
return True
# Check if output_config is directly provided
output_config = optional_params.get("output_config")
if output_config and isinstance(output_config, dict):
effort = output_config.get("effort")
if effort and isinstance(effort, str):
return True
return False
def is_code_execution_tool_used(self, tools: Optional[List]) -> bool:
"""
Check if code execution tool is being used.
Returns True if any tool has type "code_execution_20250825".
"""
if not tools:
return False
for tool in tools:
tool_type = tool.get("type", "")
if tool_type == "code_execution_20250825":
return True
return False
def is_container_with_skills_used(self, optional_params: Optional[dict]) -> bool:
"""
Check if container with skills is being used.
Returns True if optional_params contains container with skills.
"""
if not optional_params:
return False
container = optional_params.get("container")
if container and isinstance(container, dict):
skills = container.get("skills")
@ -256,10 +282,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
def get_computer_tool_beta_header(self, computer_tool_version: str) -> str:
"""
Get the appropriate beta header for a given computer tool version.
Args:
computer_tool_version: The computer tool version (e.g., 'computer_20250124', 'computer_20241022')
Returns:
The corresponding beta header string
"""
@ -282,37 +308,37 @@ class AnthropicModelInfo(BaseLLMModelInfo):
) -> List[str]:
"""
Get list of common beta headers based on the features that are active.
Returns:
List of beta header strings
"""
from litellm.types.llms.anthropic import (
ANTHROPIC_EFFORT_BETA_HEADER,
)
betas = []
# Detect features
effort_used = self.is_effort_used(optional_params, model)
if effort_used:
betas.append(ANTHROPIC_EFFORT_BETA_HEADER) # effort-2025-11-24
if computer_tool_used:
beta_header = self.get_computer_tool_beta_header(computer_tool_used)
betas.append(beta_header)
# Anthropic no longer requires the prompt-caching beta header
# Prompt caching now works automatically when cache_control is used in messages
# Reference: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
if file_id_used:
betas.append("files-api-2025-04-14")
betas.append("code-execution-2025-05-22")
if mcp_server_used:
betas.append("mcp-client-2025-04-04")
return list(set(betas))
def get_anthropic_headers(
@ -351,27 +377,35 @@ class AnthropicModelInfo(BaseLLMModelInfo):
# Tool search, programmatic tool calling, and input_examples all use the same beta header
if tool_search_used or programmatic_tool_calling_used or input_examples_used:
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
betas.add(ANTHROPIC_TOOL_SEARCH_BETA_HEADER)
# Effort parameter uses a separate beta header
if effort_used:
from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER
betas.add(ANTHROPIC_EFFORT_BETA_HEADER)
# Code execution tool uses a separate beta header
if code_execution_tool_used:
betas.add("code-execution-2025-08-25")
# Container with skills uses a separate beta header
if container_with_skills_used:
betas.add("skills-2025-10-02")
_is_oauth = api_key and api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX)
headers = {
"anthropic-version": anthropic_version or "2023-06-01",
"x-api-key": api_key,
"accept": "application/json",
"content-type": "application/json",
}
if _is_oauth:
headers["authorization"] = f"Bearer {api_key}"
headers["anthropic-dangerous-direct-browser-access"] = "true"
betas.add(ANTHROPIC_OAUTH_BETA_HEADER)
else:
headers["x-api-key"] = api_key
if user_anthropic_beta_headers is not None:
betas.update(user_anthropic_beta_headers)
@ -381,7 +415,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
# Vertex AI requires web search beta header for web search to work
if web_search_tool_used:
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
headers[
"anthropic-beta"
] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
elif len(betas) > 0:
headers["anthropic-beta"] = ",".join(betas)
@ -398,7 +435,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
api_base: Optional[str] = None,
) -> Dict:
# Check for Anthropic OAuth token in headers
headers, api_key = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key)
headers, api_key = optionally_handle_anthropic_oauth(
headers=headers, api_key=api_key
)
if api_key is None:
raise litellm.AuthenticationError(
message="Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params. Please set `ANTHROPIC_API_KEY` in your environment vars",
@ -416,11 +455,15 @@ class AnthropicModelInfo(BaseLLMModelInfo):
file_id_used = self.is_file_id_used(messages=messages)
web_search_tool_used = self.is_web_search_tool_used(tools=tools)
tool_search_used = self.is_tool_search_used(tools=tools)
programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools)
programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(
tools=tools
)
input_examples_used = self.is_input_examples_used(tools=tools)
effort_used = self.is_effort_used(optional_params=optional_params, model=model)
code_execution_tool_used = self.is_code_execution_tool_used(tools=tools)
container_with_skills_used = self.is_container_with_skills_used(optional_params=optional_params)
container_with_skills_used = self.is_container_with_skills_used(
optional_params=optional_params
)
user_anthropic_beta_headers = self._get_user_anthropic_beta_headers(
anthropic_beta_header=headers.get("anthropic-beta")
)
@ -499,7 +542,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
def get_token_counter(self) -> Optional[BaseTokenCounter]:
"""
Factory method to create an Anthropic token counter.
Returns:
AnthropicTokenCounter instance for this provider.
"""

34
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -49,15 +49,15 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
# TODO: Add Anthropic `metadata` support
# "metadata",
]
@staticmethod
def _filter_billing_headers_from_system(system_param):
"""
Filter out x-anthropic-billing-header metadata from system parameter.
Args:
system_param: Can be a string or a list of system message content blocks
Returns:
Filtered system parameter (string or list), or None if all content was filtered
"""
@ -74,7 +74,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
text = content_block.get("text", "")
content_type = content_block.get("type", "")
# Skip text blocks that start with billing header
if content_type == "text" and text.startswith("x-anthropic-billing-header:"):
if content_type == "text" and text.startswith(
"x-anthropic-billing-header:"
):
continue
filtered_list.append(content_block)
else:
@ -111,11 +113,13 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
import os
# Check for Anthropic OAuth token in Authorization header
headers, api_key = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key)
headers, api_key = optionally_handle_anthropic_oauth(
headers=headers, api_key=api_key
)
if api_key is None:
api_key = os.getenv("ANTHROPIC_API_KEY")
if "x-api-key" not in headers and api_key:
if "x-api-key" not in headers and "authorization" not in headers and api_key:
headers["x-api-key"] = api_key
if "anthropic-version" not in headers:
headers["anthropic-version"] = DEFAULT_ANTHROPIC_API_VERSION
@ -149,7 +153,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
message="max_tokens is required for Anthropic /v1/messages API",
status_code=400,
)
# Filter out x-anthropic-billing-header from system messages
system_param = anthropic_messages_optional_request_params.get("system")
if system_param is not None:
@ -159,7 +163,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
else:
# Remove system parameter if all content was filtered out
anthropic_messages_optional_request_params.pop("system", None)
####### get required params for all anthropic messages requests ######
verbose_logger.debug(f"TRANSFORMATION DEBUG - Messages: {messages}")
anthropic_messages_request: AnthropicMessagesRequest = AnthropicMessagesRequest(
@ -244,25 +248,29 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
edits = context_management_param.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.add(
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
)
# Check for structured outputs
if optional_params.get("output_format") is not None:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value)
beta_values.add(
ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value
)
# Check for fast mode
if optional_params.get("speed") == "fast":

30
litellm/llms/azure/azure.py
View file

@ -901,7 +901,20 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
if response.json()["status"] == "failed":
error_data = response.json()
raise AzureOpenAIError(status_code=400, message=json.dumps(error_data))
# Preserve Azure error details (e.g. content_policy_violation,
# inner_error, content_filter_results) as structured body so
# exception_type() can route them correctly.
_error_body = error_data.get("error", error_data)
_error_msg = (
_error_body.get("message", "Image generation failed")
if isinstance(_error_body, dict)
else json.dumps(error_data)
)
raise AzureOpenAIError(
status_code=400,
message=_error_msg,
body=error_data,
)
result = response.json()["result"]
return httpx.Response(
@ -999,7 +1012,20 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
if response.json()["status"] == "failed":
error_data = response.json()
raise AzureOpenAIError(status_code=400, message=json.dumps(error_data))
# Preserve Azure error details (e.g. content_policy_violation,
# inner_error, content_filter_results) as structured body so
# exception_type() can route them correctly.
_error_body = error_data.get("error", error_data)
_error_msg = (
_error_body.get("message", "Image generation failed")
if isinstance(_error_body, dict)
else json.dumps(error_data)
)
raise AzureOpenAIError(
status_code=400,
message=_error_msg,
body=error_data,
)
result = response.json()["result"]
return httpx.Response(

17
litellm/llms/azure/responses/transformation.py
View file

@ -1,5 +1,5 @@
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union
from copy import deepcopy
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union
import httpx
from openai.types.responses import ResponseReasoningItem
@ -21,10 +21,25 @@ else:
class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Parameters not supported by Azure Responses API
AZURE_UNSUPPORTED_PARAMS = ["context_management"]
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.AZURE
def get_supported_openai_params(self, model: str) -> list:
"""
Azure Responses API does not support context_management (compaction).
"""
base_supported_params = super().get_supported_openai_params(model)
return [
param
for param in base_supported_params
if param not in self.AZURE_UNSUPPORTED_PARAMS
]
def validate_environment(
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:

41
litellm/llms/base_llm/managed_resources/__init__.py Normal file
View file

@ -0,0 +1,41 @@
"""
Managed Resources Module
This module provides base classes and utilities for managing resources
(files, vector stores, etc.) with target_model_names support.
The BaseManagedResource class provides common functionality for:
- Storing unified resource IDs with model mappings
- Retrieving resources by unified ID
- Deleting resources across multiple models
- Creating resources for multiple models
- Filtering deployments based on model mappings
"""
from .base_managed_resource import BaseManagedResource
from .utils import (
decode_unified_id,
encode_unified_id,
extract_model_id_from_unified_id,
extract_provider_resource_id_from_unified_id,
extract_resource_type_from_unified_id,
extract_target_model_names_from_unified_id,
extract_unified_uuid_from_unified_id,
generate_unified_id_string,
is_base64_encoded_unified_id,
parse_unified_id,
)
__all__ = [
"BaseManagedResource",
"is_base64_encoded_unified_id",
"extract_target_model_names_from_unified_id",
"extract_resource_type_from_unified_id",
"extract_unified_uuid_from_unified_id",
"extract_model_id_from_unified_id",
"extract_provider_resource_id_from_unified_id",
"generate_unified_id_string",
"encode_unified_id",
"decode_unified_id",
"parse_unified_id",
]

605
litellm/llms/base_llm/managed_resources/base_managed_resource.py Normal file
View file

@ -0,0 +1,605 @@
# What is this?
## Base class for managing resources (files, vector stores, etc.) with target_model_names support
## This provides common functionality for creating, retrieving, and managing resources across multiple models
import base64
import json
from abc import ABC, abstractmethod
from typing import (
TYPE_CHECKING,
Any,
Dict,
Generic,
List,
Optional,
TypeVar,
Union,
cast,
)
from litellm import verbose_logger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import SpecialEnums
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
from litellm.router import Router as _Router
Span = Union[_Span, Any]
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
Router = _Router
else:
Span = Any
InternalUsageCache = Any
PrismaClient = Any
Router = Any
# Generic type for resource objects
ResourceObjectType = TypeVar('ResourceObjectType')
class BaseManagedResource(ABC, Generic[ResourceObjectType]):
"""
Base class for managing resources with target_model_names support.
This class provides common functionality for:
- Storing unified resource IDs with model mappings
- Retrieving resources by unified ID
- Deleting resources across multiple models
- Creating resources for multiple models
- Filtering deployments based on model mappings
Subclasses should implement:
- resource_type: str property
- table_name: str property
- create_resource_for_model: method to create resource on a specific model
- get_unified_resource_id_format: method to generate unified ID format
"""
def __init__(
self,
internal_usage_cache: InternalUsageCache,
prisma_client: PrismaClient,
):
self.internal_usage_cache = internal_usage_cache
self.prisma_client = prisma_client
# ============================================================================
# ABSTRACT METHODS
# ============================================================================
@property
@abstractmethod
def resource_type(self) -> str:
"""
Return the resource type identifier (e.g., 'file', 'vector_store', 'vector_store_file').
Used for logging and unified ID generation.
"""
pass
@property
@abstractmethod
def table_name(self) -> str:
"""
Return the database table name for this resource type.
Example: 'litellm_managedfiletable', 'litellm_managedvectorstoretable'
"""
pass
@abstractmethod
def get_unified_resource_id_format(
self,
resource_object: ResourceObjectType,
target_model_names_list: List[str],
) -> str:
"""
Generate the format string for the unified resource ID.
This should return a string that will be base64 encoded.
Example for files:
"litellm_proxy:application/json;unified_id,{uuid};target_model_names,{models};..."
Args:
resource_object: The resource object returned from the provider
target_model_names_list: List of target model names
Returns:
Format string to be base64 encoded
"""
pass
@abstractmethod
async def create_resource_for_model(
self,
llm_router: Router,
model: str,
request_data: Dict[str, Any],
litellm_parent_otel_span: Span,
) -> ResourceObjectType:
"""
Create a resource for a specific model.
Args:
llm_router: LiteLLM router instance
model: Model name to create resource for
request_data: Request data for resource creation
litellm_parent_otel_span: OpenTelemetry span for tracing
Returns:
Resource object from the provider
"""
pass
# ============================================================================
# COMMON STORAGE OPERATIONS
# ============================================================================
async def store_unified_resource_id(
self,
unified_resource_id: str,
resource_object: Optional[ResourceObjectType],
litellm_parent_otel_span: Optional[Span],
model_mappings: Dict[str, str],
user_api_key_dict: UserAPIKeyAuth,
additional_db_fields: Optional[Dict[str, Any]] = None,
) -> None:
"""
Store unified resource ID with model mappings in cache and database.
Args:
unified_resource_id: The unified resource ID (base64 encoded)
resource_object: The resource object to store (can be None)
litellm_parent_otel_span: OpenTelemetry span for tracing
model_mappings: Dictionary mapping model_id -> provider_resource_id
user_api_key_dict: User API key authentication details
additional_db_fields: Additional fields to store in database
"""
verbose_logger.info(
f"Storing LiteLLM Managed {self.resource_type} with id={unified_resource_id} in cache"
)
# Prepare cache data
cache_data = {
"unified_resource_id": unified_resource_id,
"resource_object": resource_object,
"model_mappings": model_mappings,
"flat_model_resource_ids": list(model_mappings.values()),
"created_by": user_api_key_dict.user_id,
"updated_by": user_api_key_dict.user_id,
}
# Add additional fields if provided
if additional_db_fields:
cache_data.update(additional_db_fields)
# Store in cache
if resource_object is not None:
await self.internal_usage_cache.async_set_cache(
key=unified_resource_id,
value=cache_data,
litellm_parent_otel_span=litellm_parent_otel_span,
)
# Prepare database data
db_data = {
"unified_resource_id": unified_resource_id,
"model_mappings": json.dumps(model_mappings),
"flat_model_resource_ids": list(model_mappings.values()),
"created_by": user_api_key_dict.user_id,
"updated_by": user_api_key_dict.user_id,
}
# Add resource object if available
if resource_object is not None:
# Handle both dict and Pydantic models
if hasattr(resource_object, "model_dump_json"):
db_data["resource_object"] = resource_object.model_dump_json() # type: ignore
elif isinstance(resource_object, dict):
db_data["resource_object"] = json.dumps(resource_object)
# Extract storage metadata from hidden params if present
hidden_params = getattr(resource_object, "_hidden_params", {}) or {}
if "storage_backend" in hidden_params:
db_data["storage_backend"] = hidden_params["storage_backend"]
if "storage_url" in hidden_params:
db_data["storage_url"] = hidden_params["storage_url"]
# Add additional fields to database
if additional_db_fields:
db_data.update(additional_db_fields)
# Store in database
table = getattr(self.prisma_client.db, self.table_name)
result = await table.create(data=db_data)
verbose_logger.debug(
f"LiteLLM Managed {self.resource_type} with id={unified_resource_id} stored in db: {result}"
)
async def get_unified_resource_id(
self,
unified_resource_id: str,
litellm_parent_otel_span: Optional[Span] = None,
) -> Optional[Dict[str, Any]]:
"""
Retrieve unified resource by ID from cache or database.
Args:
unified_resource_id: The unified resource ID to retrieve
litellm_parent_otel_span: OpenTelemetry span for tracing
Returns:
Dictionary containing resource data or None if not found
"""
# Check cache first
result = cast(
Optional[dict],
await self.internal_usage_cache.async_get_cache(
key=unified_resource_id,
litellm_parent_otel_span=litellm_parent_otel_span,
),
)
if result:
return result
# Check database
table = getattr(self.prisma_client.db, self.table_name)
db_object = await table.find_first(
where={"unified_resource_id": unified_resource_id}
)
if db_object:
return db_object.model_dump()
return None
async def delete_unified_resource_id(
self,
unified_resource_id: str,
litellm_parent_otel_span: Optional[Span] = None,
) -> Optional[ResourceObjectType]:
"""
Delete unified resource from cache and database.
Args:
unified_resource_id: The unified resource ID to delete
litellm_parent_otel_span: OpenTelemetry span for tracing
Returns:
The deleted resource object or None if not found
"""
# Get old value from database
table = getattr(self.prisma_client.db, self.table_name)
initial_value = await table.find_first(
where={"unified_resource_id": unified_resource_id}
)
if initial_value is None:
raise Exception(
f"LiteLLM Managed {self.resource_type} with id={unified_resource_id} not found"
)
# Delete from cache
await self.internal_usage_cache.async_set_cache(
key=unified_resource_id,
value=None,
litellm_parent_otel_span=litellm_parent_otel_span,
)
# Delete from database
await table.delete(where={"unified_resource_id": unified_resource_id})
return initial_value.resource_object
async def can_user_access_unified_resource_id(
self,
unified_resource_id: str,
user_api_key_dict: UserAPIKeyAuth,
litellm_parent_otel_span: Optional[Span] = None,
) -> bool:
"""
Check if user has access to the unified resource ID.
Uses get_unified_resource_id() which checks cache first before hitting the database,
avoiding direct DB queries in the critical request path.
Args:
unified_resource_id: The unified resource ID to check
user_api_key_dict: User API key authentication details
litellm_parent_otel_span: OpenTelemetry span for tracing
Returns:
True if user has access, False otherwise
"""
user_id = user_api_key_dict.user_id
# Use cached method instead of direct DB query
resource = await self.get_unified_resource_id(
unified_resource_id, litellm_parent_otel_span
)
if resource:
return resource.get("created_by") == user_id
return False
# ============================================================================
# MODEL MAPPING OPERATIONS
# ============================================================================
async def get_model_resource_id_mapping(
self,
resource_ids: List[str],
litellm_parent_otel_span: Span,
) -> Dict[str, Dict[str, str]]:
"""
Get model-specific resource IDs for a list of unified resource IDs.
Args:
resource_ids: List of unified resource IDs
litellm_parent_otel_span: OpenTelemetry span for tracing
Returns:
Dictionary mapping unified_resource_id -> model_id -> provider_resource_id
Example:
{
"unified_resource_id_1": {
"model_id_1": "provider_resource_id_1",
"model_id_2": "provider_resource_id_2"
}
}
"""
resource_id_mapping: Dict[str, Dict[str, str]] = {}
for resource_id in resource_ids:
# Get unified resource from cache/db
unified_resource_object = await self.get_unified_resource_id(
resource_id, litellm_parent_otel_span
)
if unified_resource_object:
model_mappings = unified_resource_object.get("model_mappings", {})
# Handle both JSON string and dict
if isinstance(model_mappings, str):
model_mappings = json.loads(model_mappings)
resource_id_mapping[resource_id] = model_mappings
return resource_id_mapping
# ============================================================================
# RESOURCE CREATION OPERATIONS
# ============================================================================
async def create_resource_for_each_model(
self,
llm_router: Router,
request_data: Dict[str, Any],
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
) -> List[ResourceObjectType]:
"""
Create a resource for each model in the target list.
Args:
llm_router: LiteLLM router instance
request_data: Request data for resource creation
target_model_names_list: List of target model names
litellm_parent_otel_span: OpenTelemetry span for tracing
Returns:
List of resource objects created for each model
"""
if llm_router is None:
raise Exception("LLM Router not initialized. Ensure models added to proxy.")
responses = []
for model in target_model_names_list:
individual_response = await self.create_resource_for_model(
llm_router=llm_router,
model=model,
request_data=request_data,
litellm_parent_otel_span=litellm_parent_otel_span,
)
responses.append(individual_response)
return responses
def generate_unified_resource_id(
self,
resource_objects: List[ResourceObjectType],
target_model_names_list: List[str],
) -> str:
"""
Generate a unified resource ID from multiple resource objects.
Args:
resource_objects: List of resource objects from different models
target_model_names_list: List of target model names
Returns:
Base64 encoded unified resource ID
"""
# Use the first resource object to generate the format
unified_id_format = self.get_unified_resource_id_format(
resource_object=resource_objects[0],
target_model_names_list=target_model_names_list,
)
# Convert to URL-safe base64 and strip padding
base64_unified_id = (
base64.urlsafe_b64encode(unified_id_format.encode()).decode().rstrip("=")
)
return base64_unified_id
def extract_model_mappings_from_responses(
self,
resource_objects: List[ResourceObjectType],
) -> Dict[str, str]:
"""
Extract model mappings from resource objects.
Args:
resource_objects: List of resource objects from different models
Returns:
Dictionary mapping model_id -> provider_resource_id
"""
model_mappings: Dict[str, str] = {}
for resource_object in resource_objects:
# Get hidden params if available
hidden_params = getattr(resource_object, "_hidden_params", {}) or {}
model_resource_id_mapping = hidden_params.get("model_resource_id_mapping")
if model_resource_id_mapping and isinstance(model_resource_id_mapping, dict):
model_mappings.update(model_resource_id_mapping)
return model_mappings
# ============================================================================
# DEPLOYMENT FILTERING
# ============================================================================
async def async_filter_deployments(
self,
model: str,
healthy_deployments: List,
request_kwargs: Optional[Dict] = None,
parent_otel_span: Optional[Span] = None,
resource_id_key: str = "resource_id",
) -> List[Dict]:
"""
Filter deployments based on model mappings for a resource.
This is used by the router to select only deployments that have
the resource available.
Args:
model: Model name
healthy_deployments: List of healthy deployments
request_kwargs: Request kwargs containing resource_id and mappings
parent_otel_span: OpenTelemetry span for tracing
resource_id_key: Key to use for resource ID in request_kwargs
Returns:
Filtered list of deployments
"""
if request_kwargs is None:
return healthy_deployments
resource_id = cast(Optional[str], request_kwargs.get(resource_id_key))
model_resource_id_mapping = cast(
Optional[Dict[str, Dict[str, str]]],
request_kwargs.get("model_resource_id_mapping"),
)
allowed_model_ids = []
if resource_id and model_resource_id_mapping:
model_id_dict = model_resource_id_mapping.get(resource_id, {})
allowed_model_ids = list(model_id_dict.keys())
if len(allowed_model_ids) == 0:
return healthy_deployments
return [
deployment
for deployment in healthy_deployments
if deployment.get("model_info", {}).get("id") in allowed_model_ids
]
# ============================================================================
# UTILITY METHODS
# ============================================================================
def get_unified_id_prefix(self) -> str:
"""
Get the prefix for unified IDs for this resource type.
Returns:
Prefix string (e.g., "litellm_proxy:")
"""
return SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value
async def list_user_resources(
self,
user_api_key_dict: UserAPIKeyAuth,
limit: Optional[int] = None,
after: Optional[str] = None,
additional_filters: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""
List resources created by a user.
Args:
user_api_key_dict: User API key authentication details
limit: Maximum number of resources to return
after: Cursor for pagination
additional_filters: Additional filters to apply
Returns:
Dictionary with list of resources and pagination info
"""
where_clause: Dict[str, Any] = {}
# Filter by user who created the resource
if user_api_key_dict.user_id:
where_clause["created_by"] = user_api_key_dict.user_id
if after:
where_clause["id"] = {"gt": after}
# Add additional filters
if additional_filters:
where_clause.update(additional_filters)
# Fetch resources
fetch_limit = limit or 20
table = getattr(self.prisma_client.db, self.table_name)
resources = await table.find_many(
where=where_clause,
take=fetch_limit,
order={"created_at": "desc"},
)
resource_objects: List[Any] = []
for resource in resources:
try:
# Stop once we have enough
if len(resource_objects) >= (limit or 20):
break
# Parse resource object
resource_data = resource.resource_object
if isinstance(resource_data, str):
resource_data = json.loads(resource_data)
# Set unified ID
if hasattr(resource_data, "id"):
resource_data.id = resource.unified_resource_id
elif isinstance(resource_data, dict):
resource_data["id"] = resource.unified_resource_id
resource_objects.append(resource_data)
except Exception as e:
verbose_logger.warning(
f"Failed to parse {self.resource_type} object "
f"{resource.unified_resource_id}: {e}"
)
continue
return {
"object": "list",
"data": resource_objects,
"first_id": resource_objects[0].id if resource_objects else None,
"last_id": resource_objects[-1].id if resource_objects else None,
"has_more": len(resource_objects) == (limit or 20),
}

364
litellm/llms/base_llm/managed_resources/utils.py Normal file
View file

@ -0,0 +1,364 @@
"""
Utility functions for managed resources.
This module provides common utility functions that can be used across
different managed resource types (files, vector stores, etc.).
"""
import base64
import re
from typing import List, Optional, Union, Literal
def is_base64_encoded_unified_id(
resource_id: str,
prefix: str = "litellm_proxy:",
) -> Union[str, Literal[False]]:
"""
Check if a resource ID is a base64 encoded unified ID.
Args:
resource_id: The resource ID to check
prefix: The expected prefix for unified IDs
Returns:
Decoded string if valid unified ID, False otherwise
"""
# Ensure resource_id is a string
if not isinstance(resource_id, str):
return False
# Add padding back if needed
padded = resource_id + "=" * (-len(resource_id) % 4)
# Decode from base64
try:
decoded = base64.urlsafe_b64decode(padded).decode()
if decoded.startswith(prefix):
return decoded
else:
return False
except Exception:
return False
def extract_target_model_names_from_unified_id(
unified_id: str,
) -> List[str]:
"""
Extract target model names from a unified resource ID.
Args:
unified_id: The unified resource ID (decoded or encoded)
Returns:
List of target model names
Example:
unified_id = "litellm_proxy:vector_store;unified_id,uuid;target_model_names,gpt-4,gemini-2.0"
returns: ["gpt-4", "gemini-2.0"]
"""
try:
# Ensure unified_id is a string
if not isinstance(unified_id, str):
return []
# Decode if it's base64 encoded
decoded_id = is_base64_encoded_unified_id(unified_id)
if decoded_id:
unified_id = decoded_id
# Extract model names using regex
match = re.search(r"target_model_names,([^;]+)", unified_id)
if match:
# Split on comma and strip whitespace from each model name
return [model.strip() for model in match.group(1).split(",")]
return []
except Exception:
return []
def extract_resource_type_from_unified_id(
unified_id: str,
) -> Optional[str]:
"""
Extract resource type from a unified resource ID.
Args:
unified_id: The unified resource ID (decoded or encoded)
Returns:
Resource type string or None
Example:
unified_id = "litellm_proxy:vector_store;unified_id,uuid;..."
returns: "vector_store"
"""
try:
# Ensure unified_id is a string
if not isinstance(unified_id, str):
return None
# Decode if it's base64 encoded
decoded_id = is_base64_encoded_unified_id(unified_id)
if decoded_id:
unified_id = decoded_id
# Extract resource type (comes after prefix and before first semicolon)
match = re.search(r"litellm_proxy:([^;]+)", unified_id)
if match:
return match.group(1).strip()
return None
except Exception:
return None
def extract_unified_uuid_from_unified_id(
unified_id: str,
) -> Optional[str]:
"""
Extract the UUID from a unified resource ID.
Args:
unified_id: The unified resource ID (decoded or encoded)
Returns:
UUID string or None
Example:
unified_id = "litellm_proxy:vector_store;unified_id,abc-123;..."
returns: "abc-123"
"""
try:
# Ensure unified_id is a string
if not isinstance(unified_id, str):
return None
# Decode if it's base64 encoded
decoded_id = is_base64_encoded_unified_id(unified_id)
if decoded_id:
unified_id = decoded_id
# Extract UUID
match = re.search(r"unified_id,([^;]+)", unified_id)
if match:
return match.group(1).strip()
return None
except Exception:
return None
def extract_model_id_from_unified_id(
unified_id: str,
) -> Optional[str]:
"""
Extract model ID from a unified resource ID.
Args:
unified_id: The unified resource ID (decoded or encoded)
Returns:
Model ID string or None
Example:
unified_id = "litellm_proxy:vector_store;...;model_id,gpt-4-model-id;..."
returns: "gpt-4-model-id"
"""
try:
# Ensure unified_id is a string
if not isinstance(unified_id, str):
return None
# Decode if it's base64 encoded
decoded_id = is_base64_encoded_unified_id(unified_id)
if decoded_id:
unified_id = decoded_id
# Extract model ID
match = re.search(r"model_id,([^;]+)", unified_id)
if match:
return match.group(1).strip()
return None
except Exception:
return None
def extract_provider_resource_id_from_unified_id(
unified_id: str,
) -> Optional[str]:
"""
Extract provider resource ID from a unified resource ID.
Args:
unified_id: The unified resource ID (decoded or encoded)
Returns:
Provider resource ID string or None
Example:
unified_id = "litellm_proxy:vector_store;...;resource_id,vs_abc123;..."
returns: "vs_abc123"
"""
try:
# Ensure unified_id is a string
if not isinstance(unified_id, str):
return None
# Decode if it's base64 encoded
decoded_id = is_base64_encoded_unified_id(unified_id)
if decoded_id:
unified_id = decoded_id
# Extract resource ID (try multiple patterns for different resource types)
patterns = [
r"resource_id,([^;]+)",
r"vector_store_id,([^;]+)",
r"file_id,([^;]+)",
]
for pattern in patterns:
match = re.search(pattern, unified_id)
if match:
return match.group(1).strip()
return None
except Exception:
return None
def generate_unified_id_string(
resource_type: str,
unified_uuid: str,
target_model_names: List[str],
provider_resource_id: str,
model_id: str,
additional_fields: Optional[dict] = None,
) -> str:
"""
Generate a unified ID string (before base64 encoding).
Args:
resource_type: Type of resource (e.g., "vector_store", "file")
unified_uuid: UUID for this unified resource
target_model_names: List of target model names
provider_resource_id: Resource ID from the provider
model_id: Model ID from the router
additional_fields: Additional fields to include in the ID
Returns:
Unified ID string (not yet base64 encoded)
Example:
generate_unified_id_string(
resource_type="vector_store",
unified_uuid="abc-123",
target_model_names=["gpt-4", "gemini"],
provider_resource_id="vs_xyz",
model_id="model-id-123",
)
returns: "litellm_proxy:vector_store;unified_id,abc-123;target_model_names,gpt-4,gemini;resource_id,vs_xyz;model_id,model-id-123"
"""
# Build the unified ID string
parts = [
f"litellm_proxy:{resource_type}",
f"unified_id,{unified_uuid}",
f"target_model_names,{','.join(target_model_names)}",
f"resource_id,{provider_resource_id}",
f"model_id,{model_id}",
]
# Add additional fields if provided
if additional_fields:
for key, value in additional_fields.items():
parts.append(f"{key},{value}")
return ";".join(parts)
def encode_unified_id(unified_id_string: str) -> str:
"""
Encode a unified ID string to base64.
Args:
unified_id_string: The unified ID string to encode
Returns:
Base64 encoded unified ID (URL-safe, padding stripped)
"""
return (
base64.urlsafe_b64encode(unified_id_string.encode())
.decode()
.rstrip("=")
)
def decode_unified_id(encoded_unified_id: str) -> Optional[str]:
"""
Decode a base64 encoded unified ID.
Args:
encoded_unified_id: The base64 encoded unified ID
Returns:
Decoded unified ID string or None if invalid
"""
try:
# Add padding back if needed
padded = encoded_unified_id + "=" * (-len(encoded_unified_id) % 4)
# Decode from base64
decoded = base64.urlsafe_b64decode(padded).decode()
# Verify it starts with the expected prefix
if decoded.startswith("litellm_proxy:"):
return decoded
return None
except Exception:
return None
def parse_unified_id(
unified_id: str,
) -> Optional[dict]:
"""
Parse a unified ID into its components.
Args:
unified_id: The unified ID (encoded or decoded)
Returns:
Dictionary with parsed components or None if invalid
Example:
{
"resource_type": "vector_store",
"unified_uuid": "abc-123",
"target_model_names": ["gpt-4", "gemini"],
"provider_resource_id": "vs_xyz",
"model_id": "model-id-123"
}
"""
try:
# Decode if needed
decoded_id = decode_unified_id(unified_id)
if not decoded_id:
# Maybe it's already decoded
if unified_id.startswith("litellm_proxy:"):
decoded_id = unified_id
else:
return None
return {
"resource_type": extract_resource_type_from_unified_id(decoded_id),
"unified_uuid": extract_unified_uuid_from_unified_id(decoded_id),
"target_model_names": extract_target_model_names_from_unified_id(decoded_id),
"provider_resource_id": extract_provider_resource_id_from_unified_id(decoded_id),
"model_id": extract_model_id_from_unified_id(decoded_id),
}
except Exception:
return None

33
litellm/llms/custom_httpx/aiohttp_transport.py
View file

@ -4,7 +4,7 @@ import os
import ssl
import typing
import urllib.request
from typing import Callable, Dict, Optional, Union
from typing import Any, Callable, Dict, Optional, Union
import aiohttp
import aiohttp.client_exceptions
@ -248,26 +248,25 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
# Only pass ssl kwarg when explicitly configured, to avoid
# overriding the session/connector defaults with None (which is
# not a valid value for aiohttp's ssl parameter).
ssl_kwargs: Dict[str, Union[bool, ssl.SSLContext]] = {}
if ssl_verify is not None:
ssl_kwargs["ssl"] = ssl_verify
response = await client_session.request(
method=request.method,
url=YarlURL(str(request.url), encoded=True),
headers=request.headers,
data=data,
allow_redirects=False,
auto_decompress=False,
timeout=ClientTimeout(
request_kwargs: Dict[str, Any] = {
"method": request.method,
"url": YarlURL(str(request.url), encoded=True),
"headers": request.headers,
"data": data,
"allow_redirects": False,
"auto_decompress": False,
"timeout": ClientTimeout(
sock_connect=timeout.get("connect"),
sock_read=timeout.get("read"),
connect=timeout.get("pool"),
),
proxy=proxy,
server_hostname=sni_hostname,
**ssl_kwargs,
).__aenter__()
"proxy": proxy,
"server_hostname": sni_hostname,
}
if ssl_verify is not None:
request_kwargs["ssl"] = ssl_verify
response = await client_session.request(**request_kwargs).__aenter__()
return response

23
litellm/llms/custom_httpx/http_handler.py
View file

@ -1206,7 +1206,28 @@ def get_async_httpx_client(
If not present, creates a new client
Caches the new client and returns it.
Note: When shared_session is provided, the cache is bypassed to ensure
the user's session (with its trace_configs, connector settings, etc.)
is used for the request.
"""
# When shared_session is provided, bypass cache and create a new handler
# that uses the user's session directly. This preserves the user's
# session configuration including trace_configs for aiohttp tracing.
if shared_session is not None:
verbose_logger.debug(
f"shared_session provided (ID: {id(shared_session)}), bypassing client cache"
)
if params is not None:
handler_params = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"}
handler_params["shared_session"] = shared_session
return AsyncHTTPHandler(**handler_params)
else:
return AsyncHTTPHandler(
timeout=httpx.Timeout(timeout=600.0, connect=5.0),
shared_session=shared_session,
)
_params_key_name = ""
if params is not None:
for key, value in params.items():
@ -1233,12 +1254,10 @@ def get_async_httpx_client(
if params is not None:
# Filter out params that are only used for cache key, not for AsyncHTTPHandler.__init__
handler_params = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"}
handler_params["shared_session"] = shared_session
_new_client = AsyncHTTPHandler(**handler_params)
else:
_new_client = AsyncHTTPHandler(
timeout=httpx.Timeout(timeout=600.0, connect=5.0),
shared_session=shared_session,
)
cache.set_cache(

17
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -772,12 +772,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
try:
return ModelResponseStream(
id=chunk["id"],
object="chat.completion.chunk",
created=chunk.get("created"),
model=chunk.get("model"),
choices=chunk.get("choices", []),
)
kwargs = {
"id": chunk["id"],
"object": "chat.completion.chunk",
"created": chunk.get("created"),
"model": chunk.get("model"),
"choices": chunk.get("choices", []),
}
if "usage" in chunk and chunk["usage"] is not None:
kwargs["usage"] = chunk["usage"]
return ModelResponseStream(**kwargs)
except Exception as e:
raise e

27
litellm/llms/openai/responses/transformation.py
View file

@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast, get_type_hin
import httpx
from openai.types.responses import ResponseReasoningItem
from pydantic import BaseModel
from pydantic import BaseModel, ValidationError
import litellm
from litellm._logging import verbose_logger
@ -240,25 +240,26 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
event_pydantic_model = OpenAIResponsesAPIConfig.get_event_model_class(
event_type=event_type
)
# Defensive: Some OpenAI-compatible providers may send `error.code: null`.
# Pydantic will raise a ValidationError when it expects a string but gets None.
# Coalesce a None `error.code` to a stable default string so streaming
# iteration does not crash (see issue report). This keeps behavior similar
# to previous fixes (coalesce before validation) and lets higher-level
# handlers still receive an `ErrorEvent` object.
# Some OpenAI-compatible providers send error.code: null; coalesce so validation succeeds.
try:
error_obj = parsed_chunk.get("error")
if isinstance(error_obj, dict) and error_obj.get("code") is None:
# Preserve other fields, but ensure `code` is a non-null string
parsed_chunk = dict(parsed_chunk)
parsed_chunk["error"] = dict(error_obj)
parsed_chunk["error"]["code"] = "unknown_error"
except Exception:
# If anything unexpected happens here, fall back to attempting
# instantiation and let higher-level handlers manage errors.
verbose_logger.debug("Failed to coalesce error.code in parsed_chunk")
return event_pydantic_model(**parsed_chunk)
try:
return event_pydantic_model(**parsed_chunk)
except ValidationError:
verbose_logger.debug(
"Pydantic validation failed for %s with chunk %s, "
"falling back to model_construct",
event_pydantic_model.__name__,
parsed_chunk,
)
return event_pydantic_model.model_construct(**parsed_chunk)
@staticmethod
def get_event_model_class(event_type: str) -> Any:
@ -307,6 +308,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
ResponsesAPIStreamEvents.MCP_CALL_FAILED: MCPCallFailedEvent,
ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE: ImageGenerationPartialImageEvent,
ResponsesAPIStreamEvents.ERROR: ErrorEvent,
# Shell tool events: passthrough as GenericEvent so payload is preserved
ResponsesAPIStreamEvents.SHELL_CALL_IN_PROGRESS: GenericEvent,
ResponsesAPIStreamEvents.SHELL_CALL_COMPLETED: GenericEvent,
ResponsesAPIStreamEvents.SHELL_CALL_OUTPUT: GenericEvent,
}
model_class = event_models.get(cast(ResponsesAPIStreamEvents, event_type))

4
litellm/llms/openai_like/providers.json
View file

@ -26,6 +26,10 @@
"max_completion_tokens": "max_tokens"
}
},
"scaleway": {
"base_url": "https://api.scaleway.ai/v1",
"api_key_env": "SCW_SECRET_KEY"
},
"synthetic": {
"base_url": "https://api.synthetic.new/openai/v1",
"api_key_env": "SYNTHETIC_API_KEY",

13
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -529,6 +529,18 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
raise e
def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None:
"""Pop extra_body from optional_params and shallow-merge into data, deep-merging dict values."""
extra_body: Optional[dict] = optional_params.pop("extra_body", None)
if extra_body is not None:
data_dict: dict = data # type: ignore[assignment]
for k, v in extra_body.items():
if k in data_dict and isinstance(data_dict[k], dict) and isinstance(v, dict):
data_dict[k].update(v)
else:
data_dict[k] = v
def _transform_request_body(
messages: List[AllMessageValues],
model: str,
@ -619,6 +631,7 @@ def _transform_request_body(
# Only add labels for Vertex AI endpoints (not Google GenAI/AI Studio) and only if non-empty
if labels and custom_llm_provider != LlmProviders.GEMINI:
data["labels"] = labels
_pop_and_merge_extra_body(data, optional_params)
except Exception as e:
raise e

24
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -480,7 +480,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
tool = {VertexToolName.COMPUTER_USE.value: computer_use_config}
# Handle OpenAI-style web_search and web_search_preview tools
# Transform them to Gemini's googleSearch tool
elif "type" in tool and tool["type"] in ("web_search", "web_search_preview"):
elif "type" in tool and tool["type"] in (
"web_search",
"web_search_preview",
):
verbose_logger.info(
f"Gemini: Transforming OpenAI-style '{tool['type']}' tool to googleSearch"
)
@ -1196,6 +1199,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"PROHIBITED_CONTENT": "The token generation was stopped as the response was flagged for the prohibited contents.",
"SPII": "The token generation was stopped as the response was flagged for Sensitive Personally Identifiable Information (SPII) contents.",
"IMAGE_SAFETY": "The token generation was stopped as the response was flagged for image safety reasons.",
"IMAGE_PROHIBITED_CONTENT": "The token generation was stopped as the response was flagged for prohibited image content.",
}
@staticmethod
@ -1218,6 +1222,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"SPII": "content_filter",
"MALFORMED_FUNCTION_CALL": "malformed_function_call", # openai doesn't have a way of representing this
"IMAGE_SAFETY": "content_filter",
"IMAGE_PROHIBITED_CONTENT": "content_filter",
}
def translate_exception_str(self, exception_string: str):
@ -1630,7 +1635,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
completion_image_tokens = response_tokens_details.image_tokens or 0
completion_audio_tokens = response_tokens_details.audio_tokens or 0
calculated_text_tokens = (
candidates_token_count - completion_image_tokens - completion_audio_tokens
candidates_token_count
- completion_image_tokens
- completion_audio_tokens
)
response_tokens_details.text_tokens = calculated_text_tokens
#########################################################
@ -2248,6 +2255,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
citation_metadata # older approach - maintaining to prevent regressions
)
## ADD TRAFFIC TYPE ##
traffic_type = completion_response.get("usageMetadata", {}).get(
"trafficType"
)
if traffic_type:
model_response._hidden_params.setdefault("provider_specific_fields", {})["traffic_type"] = traffic_type
except Exception as e:
raise VertexAIError(
message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format(
@ -2906,6 +2920,12 @@ class ModelResponseIterator:
PromptTokensDetailsWrapper, usage.prompt_tokens_details
).web_search_requests = web_search_requests
traffic_type = processed_chunk.get("usageMetadata", {}).get(
"trafficType"
)
if traffic_type:
model_response._hidden_params.setdefault("provider_specific_fields", {})["traffic_type"] = traffic_type
setattr(model_response, "usage", usage) # type: ignore
model_response._hidden_params["is_finished"] = False

9
litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py
View file

@ -115,8 +115,13 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
vertex_project = self.get_vertex_ai_project(litellm_params)
vertex_location = self.get_vertex_ai_location(litellm_params)
# Construct full rag corpus path
full_rag_corpus = f"projects/{vertex_project}/locations/{vertex_location}/ragCorpora/{vector_store_id}"
# Handle both full corpus path and just corpus ID
if vector_store_id.startswith("projects/"):
# Already a full path
full_rag_corpus = vector_store_id
else:
# Just the corpus ID, construct full path
full_rag_corpus = f"projects/{vertex_project}/locations/{vertex_location}/ragCorpora/{vector_store_id}"
# Build the request body for Vertex AI RAG API
request_body: Dict[str, Any] = {

104
litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py
View file

@ -1,12 +1,21 @@
import types
from typing import Any, List, Optional
from typing import Any, AsyncIterator, Iterator, List, Optional, Union
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.llms.openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
OpenAIGPTConfig,
)
from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionResponse
from litellm.types.utils import ModelResponse, Usage
from litellm.types.utils import (
Delta,
ModelResponse,
ModelResponseStream,
StreamingChoices,
Usage,
)
from ...common_utils import VertexAIError
@ -79,6 +88,18 @@ class VertexAILlama3Config(OpenAIGPTConfig):
drop_params=drop_params,
)
def get_model_response_iterator(
self,
streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
sync_stream: bool,
json_mode: Optional[bool] = False,
) -> Any:
return VertexAILlama3StreamingHandler(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)
def transform_response(
self,
model: str,
@ -124,3 +145,80 @@ class VertexAILlama3Config(OpenAIGPTConfig):
)
return model_response
class VertexAILlama3StreamingHandler(OpenAIChatCompletionStreamingHandler):
"""
Vertex AI Llama models may not include role in streaming chunk deltas.
This handler ensures the first chunk always has role="assistant".
When Vertex AI returns a single chunk with both role and finish_reason (empty response),
this handler splits it into two chunks:
1. First chunk: role="assistant", content="", finish_reason=None
2. Second chunk: role=None, content=None, finish_reason="stop"
This matches OpenAI's streaming format where the first chunk has role and
the final chunk has finish_reason but no role.
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.sent_role = False
self._pending_chunk: Optional[ModelResponseStream] = None
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
result = super().chunk_parser(chunk)
if not self.sent_role and result.choices:
delta = result.choices[0].delta
finish_reason = result.choices[0].finish_reason
# If this is both the first chunk AND the final chunk (has finish_reason),
# we need to split it into two chunks to match OpenAI format
if finish_reason is not None:
# Create a pending final chunk with finish_reason but no role
self._pending_chunk = ModelResponseStream(
id=result.id,
object="chat.completion.chunk",
created=result.created,
model=result.model,
choices=[
StreamingChoices(
index=0,
delta=Delta(content=None, role=None),
finish_reason=finish_reason,
)
],
)
# Modify current chunk to be the first chunk with role but no finish_reason
result.choices[0].finish_reason = None
delta.role = "assistant"
# Ensure content is empty string for first chunk, not None
if delta.content is None:
delta.content = ""
# Prevent downstream stream wrapper from dropping this chunk
# (it drops empty-content chunks unless special fields are present)
if delta.provider_specific_fields is None:
delta.provider_specific_fields = {}
elif delta.role is None:
delta.role = "assistant"
# If the first chunk has empty content, ensure it's still emitted
if (delta.content == "" or delta.content is None) and delta.provider_specific_fields is None:
delta.provider_specific_fields = {}
self.sent_role = True
return result
def __next__(self):
# First return any pending chunk from a previous split
if self._pending_chunk is not None:
chunk = self._pending_chunk
self._pending_chunk = None
return chunk
return super().__next__()
async def __anext__(self):
# First return any pending chunk from a previous split
if self._pending_chunk is not None:
chunk = self._pending_chunk
self._pending_chunk = None
return chunk
return await super().__anext__()

10
litellm/main.py
View file

@ -7383,6 +7383,16 @@ def stream_chunk_builder( # noqa: PLR0915
setattr(response, "usage", usage)
# Propagate provider_specific_fields from the last chunk (contains provider
# metadata like traffic_type set during streaming)
for chunk in reversed(chunks):
hidden = getattr(chunk, "_hidden_params", None)
if hidden and "provider_specific_fields" in hidden:
response._hidden_params.setdefault(
"provider_specific_fields", {}
).update(hidden["provider_specific_fields"])
break
# Add cost to usage object if include_cost_in_streaming_usage is True
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
setattr(

1893
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

678
litellm/policy_templates_backup.json Normal file
View file

@ -0,0 +1,678 @@
[
{
"id": "advanced-au-pii-protection",
"title": "Advanced PII Protection (Australia)",
"description": "Protects Australian-specific identifiers, international employee data, financial information, credentials, protected class information, and industry-specific sensitive data.",
"icon": "ShieldCheckIcon",
"iconColor": "text-purple-500",
"iconBg": "bg-purple-50",
"guardrails": [
"au-pii-tax-identifiers",
"au-pii-passports",
"international-pii-identifiers",
"contact-information-pii",
"financial-pii",
"credentials-api-keys",
"network-infrastructure-pii",
"protected-class-information"
],
"complexity": "High",
"guardrailDefinitions": [
{
"guardrail_name": "au-pii-tax-identifiers",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{
"pattern_type": "prebuilt",
"pattern_name": "au_tfn",
"action": "MASK"
},
{
"pattern_type": "prebuilt",
"pattern_name": "au_abn",
"action": "MASK"
},
{
"pattern_type": "prebuilt",
"pattern_name": "au_medicare",
"action": "MASK"
}
],
"pattern_redaction_format": "[{pattern_name}_REDACTED]"
},
"guardrail_info": {
"description": "Masks Australian Tax File Numbers, Business Numbers, and Medicare Numbers"
}
},
{
"guardrail_name": "au-pii-passports",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{
"pattern_type": "prebuilt",
"pattern_name": "passport_australia",
"action": "MASK"
}
],
"pattern_redaction_format": "[PASSPORT_REDACTED]"
},
"guardrail_info": {
"description": "Masks Australian passport numbers"
}
},
{
"guardrail_name": "international-pii-identifiers",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{"pattern_type": "prebuilt", "pattern_name": "us_ssn", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "us_ssn_no_dash", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "passport_us", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "passport_uk", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "passport_germany", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "passport_france", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "passport_netherlands", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "nl_bsn_contextual", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "passport_china", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "passport_india", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "passport_japan", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "passport_canada", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "br_cpf", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "br_cpf_unformatted", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "br_rg", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "br_cnpj", "action": "MASK"}
],
"pattern_redaction_format": "[{pattern_name}_REDACTED]"
},
"guardrail_info": {
"description": "Masks international PII identifiers including passports and national IDs"
}
},
{
"guardrail_name": "contact-information-pii",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{"pattern_type": "prebuilt", "pattern_name": "email", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "us_phone", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "br_phone_landline", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "br_phone_mobile", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "street_address", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "br_cep", "action": "MASK"}
],
"pattern_redaction_format": "[{pattern_name}_REDACTED]"
},
"guardrail_info": {
"description": "Masks contact information including emails, phone numbers, and addresses"
}
},
{
"guardrail_name": "financial-pii",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{"pattern_type": "prebuilt", "pattern_name": "visa", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "mastercard", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "amex", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "discover", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "credit_card", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "iban", "action": "MASK"}
],
"pattern_redaction_format": "[{pattern_name}_REDACTED]"
},
"guardrail_info": {
"description": "Masks financial information including credit cards and bank account numbers"
}
},
{
"guardrail_name": "credentials-api-keys",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{"pattern_type": "prebuilt", "pattern_name": "aws_access_key", "action": "BLOCK"},
{"pattern_type": "prebuilt", "pattern_name": "aws_secret_key", "action": "BLOCK"},
{"pattern_type": "prebuilt", "pattern_name": "github_token", "action": "BLOCK"},
{"pattern_type": "prebuilt", "pattern_name": "slack_token", "action": "BLOCK"},
{"pattern_type": "prebuilt", "pattern_name": "generic_api_key", "action": "BLOCK"}
],
"pattern_redaction_format": "[{pattern_name}_REDACTED]"
},
"guardrail_info": {
"description": "Blocks requests containing API keys and credentials (AWS, GitHub, Slack)"
}
},
{
"guardrail_name": "network-infrastructure-pii",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{"pattern_type": "prebuilt", "pattern_name": "ipv4", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "ipv6", "action": "MASK"}
],
"pattern_redaction_format": "[INTERNAL_IP_REDACTED]"
},
"guardrail_info": {
"description": "Masks IP addresses in requests"
}
},
{
"guardrail_name": "protected-class-information",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{"pattern_type": "prebuilt", "pattern_name": "gender_sexual_orientation", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "race_ethnicity_national_origin", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "religion", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "age_discrimination", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "disability", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "marital_family_status", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "military_status", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "public_assistance", "action": "MASK"}
],
"pattern_redaction_format": "[PROTECTED_CLASS_INFO_REDACTED]"
},
"guardrail_info": {
"description": "Masks protected class information for HR compliance and anti-discrimination"
}
}
],
"templateData": {
"policy_name": "advanced-pii-protection-australia",
"description": "Comprehensive PII detection and masking policy for Australia. Protects Australian-specific identifiers, international employee data, financial information, credentials, protected class information, and industry-specific sensitive data.",
"guardrails_add": [
"au-pii-tax-identifiers",
"au-pii-passports",
"international-pii-identifiers",
"contact-information-pii",
"financial-pii",
"credentials-api-keys",
"network-infrastructure-pii",
"protected-class-information"
],
"guardrails_remove": []
}
},
{
"id": "baseline-pii-protection",
"title": "Baseline PII Protection",
"description": "Baseline PII protection for internal tools and testing. Focuses on credentials and high-risk identifiers only. Suitable for non-sensitive internal use.",
"icon": "ShieldCheckIcon",
"iconColor": "text-blue-500",
"iconBg": "bg-blue-50",
"guardrails": [
"au-pii-tax-identifiers",
"credentials-api-keys",
"financial-pii"
],
"complexity": "Low",
"guardrailDefinitions": [
{
"guardrail_name": "au-pii-tax-identifiers",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{"pattern_type": "prebuilt", "pattern_name": "au_tfn", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "au_abn", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "au_medicare", "action": "MASK"}
],
"pattern_redaction_format": "[{pattern_name}_REDACTED]"
},
"guardrail_info": {"description": "Masks Australian Tax File Numbers, Business Numbers, and Medicare Numbers"}
},
{
"guardrail_name": "credentials-api-keys",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{"pattern_type": "prebuilt", "pattern_name": "aws_access_key", "action": "BLOCK"},
{"pattern_type": "prebuilt", "pattern_name": "aws_secret_key", "action": "BLOCK"},
{"pattern_type": "prebuilt", "pattern_name": "github_token", "action": "BLOCK"},
{"pattern_type": "prebuilt", "pattern_name": "slack_token", "action": "BLOCK"},
{"pattern_type": "prebuilt", "pattern_name": "generic_api_key", "action": "BLOCK"}
],
"pattern_redaction_format": "[{pattern_name}_REDACTED]"
},
"guardrail_info": {"description": "Blocks requests containing API keys and credentials (AWS, GitHub, Slack)"}
},
{
"guardrail_name": "financial-pii",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"patterns": [
{"pattern_type": "prebuilt", "pattern_name": "visa", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "mastercard", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "amex", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "discover", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "credit_card", "action": "MASK"},
{"pattern_type": "prebuilt", "pattern_name": "iban", "action": "MASK"}
],
"pattern_redaction_format": "[{pattern_name}_REDACTED]"
},
"guardrail_info": {"description": "Masks financial information including credit cards and bank account numbers"}
}
],
"templateData": {
"policy_name": "baseline-pii-protection",
"description": "Baseline PII protection for internal tools and testing. Focuses on credentials and high-risk identifiers only.",
"guardrails_add": [
"au-pii-tax-identifiers",
"credentials-api-keys",
"financial-pii"
],
"guardrails_remove": []
}
},
{
"id": "nsfw-content-filter-australia",
"title": "NSFW Content Filter (Australia)",
"description": "Blocks profanity, sexual content, NSFW requests, self-harm content, and child safety violations using English and Australian-specific slang. Protects against inappropriate content including sexual solicitation, explicit content, Australian profanity, self-harm, and content involving minors.",
"icon": "ShieldExclamationIcon",
"iconColor": "text-red-500",
"iconBg": "bg-red-50",
"guardrails": [
"nsfw-content-filter-english",
"nsfw-content-filter-australian",
"nsfw-self-harm-filter",
"nsfw-child-safety-filter",
"nsfw-racial-bias-filter"
],
"complexity": "Medium",
"guardrailDefinitions": [
{
"guardrail_name": "nsfw-content-filter-english",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harm_toxic_abuse",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks profanity, sexual content, slurs, and NSFW terms in English"
}
},
{
"guardrail_name": "nsfw-content-filter-australian",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harm_toxic_abuse_au",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks Australian-specific slang and profanity (root, perv, bogan, wanker, etc.)"
}
},
{
"guardrail_name": "nsfw-self-harm-filter",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harmful_self_harm",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks content related to self-harm, suicide, and eating disorders"
}
},
{
"guardrail_name": "nsfw-child-safety-filter",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harmful_child_safety",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks inappropriate content involving minors using identifier + block word combinations"
}
},
{
"guardrail_name": "nsfw-racial-bias-filter",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "bias_racial",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks racial and ethnic discrimination, hate speech, and supremacist content"
}
}
],
"templateData": {
"policy_name": "nsfw-content-filter-australia",
"description": "NSFW content filter for Australia. Blocks profanity, sexual content, inappropriate requests, self-harm content, child safety violations, and racial bias in English and Australian slang.",
"guardrails_add": [
"nsfw-content-filter-english",
"nsfw-content-filter-australian",
"nsfw-self-harm-filter",
"nsfw-child-safety-filter",
"nsfw-racial-bias-filter"
],
"guardrails_remove": []
}
},
{
"id": "nsfw-content-filter-basic",
"title": "NSFW Content Filter (Basic)",
"description": "Basic NSFW content filtering for English only. Blocks profanity, sexual content, slurs, solicitation, explicit requests, self-harm content, and child safety violations. Suitable for most applications requiring content moderation.",
"icon": "ShieldExclamationIcon",
"iconColor": "text-orange-500",
"iconBg": "bg-orange-50",
"guardrails": [
"nsfw-content-filter-english-only",
"nsfw-self-harm-filter-basic",
"nsfw-child-safety-filter-basic",
"nsfw-racial-bias-filter-basic"
],
"complexity": "Low",
"guardrailDefinitions": [
{
"guardrail_name": "nsfw-content-filter-english-only",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harm_toxic_abuse",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks profanity, sexual content, slurs, and NSFW terms. Includes 485+ keywords covering explicit content, solicitation, sexual behavior, and exploitation."
}
},
{
"guardrail_name": "nsfw-self-harm-filter-basic",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harmful_self_harm",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks content related to self-harm, suicide, and eating disorders"
}
},
{
"guardrail_name": "nsfw-child-safety-filter-basic",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harmful_child_safety",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks inappropriate content involving minors using identifier + block word combinations"
}
},
{
"guardrail_name": "nsfw-racial-bias-filter-basic",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "bias_racial",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks racial and ethnic discrimination, hate speech, and supremacist content"
}
}
],
"templateData": {
"policy_name": "nsfw-content-filter-basic",
"description": "Basic NSFW content filter. Blocks profanity, sexual content, inappropriate requests, self-harm content, child safety violations, and racial bias in English.",
"guardrails_add": [
"nsfw-content-filter-english-only",
"nsfw-self-harm-filter-basic",
"nsfw-child-safety-filter-basic",
"nsfw-racial-bias-filter-basic"
],
"guardrails_remove": []
}
},
{
"id": "nsfw-content-filter-all-regions",
"title": "NSFW Content Filter (All Regions)",
"description": "Comprehensive multi-language NSFW content filtering. Blocks profanity, sexual content, inappropriate requests, self-harm content, and child safety violations in English, Spanish, French, German, and Australian. Best for global applications.",
"icon": "ShieldExclamationIcon",
"iconColor": "text-purple-500",
"iconBg": "bg-purple-50",
"guardrails": [
"nsfw-filter-english",
"nsfw-filter-spanish",
"nsfw-filter-french",
"nsfw-filter-german",
"nsfw-filter-australian",
"nsfw-self-harm-filter-global",
"nsfw-child-safety-filter-global",
"nsfw-racial-bias-filter-global"
],
"complexity": "High",
"guardrailDefinitions": [
{
"guardrail_name": "nsfw-filter-english",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harm_toxic_abuse",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "English profanity, sexual content, slurs, and NSFW terms (485+ keywords)"
}
},
{
"guardrail_name": "nsfw-filter-spanish",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harm_toxic_abuse_es",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Spanish profanity and offensive terms (68 keywords)"
}
},
{
"guardrail_name": "nsfw-filter-french",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harm_toxic_abuse_fr",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "French profanity and offensive terms (91 keywords)"
}
},
{
"guardrail_name": "nsfw-filter-german",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harm_toxic_abuse_de",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "German profanity and offensive terms (65 keywords)"
}
},
{
"guardrail_name": "nsfw-filter-australian",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harm_toxic_abuse_au",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Australian slang and profanity (32 keywords: root, perv, bogan, wanker, etc.)"
}
},
{
"guardrail_name": "nsfw-self-harm-filter-global",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harmful_self_harm",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks content related to self-harm, suicide, and eating disorders"
}
},
{
"guardrail_name": "nsfw-child-safety-filter-global",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "harmful_child_safety",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks inappropriate content involving minors using identifier + block word combinations"
}
},
{
"guardrail_name": "nsfw-racial-bias-filter-global",
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "pre_call",
"categories": [
{
"category": "bias_racial",
"enabled": true,
"action": "BLOCK",
"severity_threshold": "medium"
}
]
},
"guardrail_info": {
"description": "Blocks racial and ethnic discrimination, hate speech, and supremacist content"
}
}
],
"templateData": {
"policy_name": "nsfw-content-filter-all-regions",
"description": "Comprehensive multi-language NSFW content filter. Blocks profanity, inappropriate content, self-harm, child safety violations, and racial bias in English, Spanish, French, German, and Australian. Total coverage: 741+ keywords across all languages plus self-harm, child safety, and racial bias protection.",
"guardrails_add": [
"nsfw-filter-english",
"nsfw-filter-spanish",
"nsfw-filter-french",
"nsfw-filter-german",
"nsfw-filter-australian",
"nsfw-self-harm-filter-global",
"nsfw-child-safety-filter-global",
"nsfw-racial-bias-filter-global"
],
"guardrails_remove": []
}
}
]

17
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -536,10 +536,25 @@ class MCPRequestHandler:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
) -> List[str]:
try:
# Get key object permission (already loaded in main auth flow)
# Get key object permission (already loaded in main auth flow, or fetch from DB)
key_object_permission = MCPRequestHandler._get_key_object_permission(
user_api_key_auth
)
if key_object_permission is None and user_api_key_auth and user_api_key_auth.object_permission_id:
from litellm.proxy.auth.auth_checks import get_object_permission
from litellm.proxy.proxy_server import (
prisma_client,
proxy_logging_obj,
user_api_key_cache,
)
if prisma_client is not None:
key_object_permission = await get_object_permission(
object_permission_id=user_api_key_auth.object_permission_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=user_api_key_auth.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
if key_object_permission is None:
return []

4
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -498,7 +498,7 @@ def _build_oauth_protected_resource_response(
)
],
"resource": resource_url,
"scopes_supported": mcp_server.scopes if mcp_server else [],
"scopes_supported": mcp_server.scopes if mcp_server and mcp_server.scopes else [],
}
@ -605,7 +605,7 @@ def _build_oauth_authorization_server_response(
"authorization_endpoint": authorization_endpoint,
"token_endpoint": token_endpoint,
"response_types_supported": ["code"],
"scopes_supported": mcp_server.scopes if mcp_server else [],
"scopes_supported": mcp_server.scopes if mcp_server and mcp_server.scopes else [],
"grant_types_supported": ["authorization_code", "refresh_token"],
"code_challenge_methods_supported": ["S256"],
"token_endpoint_auth_methods_supported": ["client_secret_post"],

329
litellm/proxy/_experimental/mcp_server/mcp_debug.py Normal file
View file

@ -0,0 +1,329 @@
"""
MCP OAuth2 Debug Headers
========================
Client-side debugging for MCP authentication flows.
When a client sends the ``x-litellm-mcp-debug: true`` header, LiteLLM
returns masked diagnostic headers in the response so operators can
troubleshoot OAuth2 issues without SSH access to the gateway.
Response headers returned (all values are masked for safety):
x-mcp-debug-inbound-auth
Which inbound auth headers were present and how they were classified.
Example: ``x-litellm-api-key=Bearer sk-12****1234``
x-mcp-debug-oauth2-token
The OAuth2 token extracted from the Authorization header (masked).
Shows ``(none)`` if absent, or flags ``SAME_AS_LITELLM_KEY`` when
the LiteLLM API key is accidentally leaking to the MCP server.
x-mcp-debug-auth-resolution
Which auth priority was used for the outbound MCP call:
``per-request-header``, ``m2m-client-credentials``, ``static-token``,
``oauth2-passthrough``, or ``no-auth``.
x-mcp-debug-outbound-url
The upstream MCP server URL that will receive the request.
x-mcp-debug-server-auth-type
The ``auth_type`` configured on the MCP server (e.g. ``oauth2``,
``bearer_token``, ``none``).
Debugging Guide
---------------
**Common issue: LiteLLM API key leaking to the MCP server**
Symptom: ``x-mcp-debug-oauth2-token`` shows ``SAME_AS_LITELLM_KEY``.
This means the ``Authorization`` header carries the LiteLLM API key and
it's being forwarded to the upstream MCP server instead of an OAuth2 token.
Fix: Move the LiteLLM key to ``x-litellm-api-key`` so the ``Authorization``
header is free for OAuth2 discovery::
# WRONG — blocks OAuth2 discovery
claude mcp add --transport http my_server http://proxy/mcp/server \\
--header "Authorization: Bearer sk-..."
# CORRECT — LiteLLM key in dedicated header, Authorization free for OAuth2
claude mcp add --transport http my_server http://proxy/mcp/server \\
--header "x-litellm-api-key: Bearer sk-..." \\
--header "x-litellm-mcp-debug: true"
**Common issue: No OAuth2 token present**
Symptom: ``x-mcp-debug-oauth2-token`` shows ``(none)`` and
``x-mcp-debug-auth-resolution`` shows ``no-auth``.
This means the client didn't go through the OAuth2 flow. Check that:
1. The ``Authorization`` header is NOT set as a static header in the client config.
2. The ``.well-known/oauth-protected-resource`` endpoint returns valid metadata.
3. The MCP server in LiteLLM config has ``auth_type: oauth2``.
**Common issue: M2M token used instead of user token**
Symptom: ``x-mcp-debug-auth-resolution`` shows ``m2m-client-credentials``.
This means the server has ``client_id``/``client_secret``/``token_url``
configured and LiteLLM is fetching a machine-to-machine token instead of
using the per-user OAuth2 token. If you want per-user tokens, remove the
client credentials from the server config.
Usage from Claude Code::
claude mcp add --transport http my_server http://proxy/mcp/server \\
--header "x-litellm-api-key: Bearer sk-..." \\
--header "x-litellm-mcp-debug: true"
Usage with curl::
curl -H "x-litellm-mcp-debug: true" \\
-H "x-litellm-api-key: Bearer sk-..." \\
http://localhost:4000/mcp/atlassian_mcp
"""
from typing import TYPE_CHECKING, Dict, List, Optional
from starlette.types import Message, Send
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
if TYPE_CHECKING:
from litellm.types.mcp_server.mcp_server_manager import MCPServer
# Header the client sends to opt into debug mode
MCP_DEBUG_REQUEST_HEADER = "x-litellm-mcp-debug"
# Prefix for all debug response headers
_RESPONSE_HEADER_PREFIX = "x-mcp-debug"
class MCPDebug:
"""
Static helper class for MCP OAuth2 debug headers.
Provides opt-in client-side diagnostics by injecting masked
authentication info into HTTP response headers.
"""
# Masker: show first 6 and last 4 chars so you can distinguish token types
# e.g. "Bearer****ef01" vs "sk-123****cdef"
_masker = SensitiveDataMasker(
sensitive_patterns={
"authorization",
"token",
"key",
"secret",
"auth",
"bearer",
},
visible_prefix=6,
visible_suffix=4,
)
@staticmethod
def _mask(value: Optional[str]) -> str:
"""Mask a single value for safe display in headers."""
if not value:
return "(none)"
return MCPDebug._masker._mask_value(value)
@staticmethod
def is_debug_enabled(headers: Dict[str, str]) -> bool:
"""
Check if the client opted into MCP debug mode.
Looks for ``x-litellm-mcp-debug: true`` (case-insensitive) in the
request headers.
"""
for key, val in headers.items():
if key.lower() == MCP_DEBUG_REQUEST_HEADER:
return val.strip().lower() in ("true", "1", "yes")
return False
@staticmethod
def resolve_auth_resolution(
server: "MCPServer",
mcp_auth_header: Optional[str],
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]],
oauth2_headers: Optional[Dict[str, str]],
) -> str:
"""
Determine which auth priority will be used for the outbound MCP call.
Returns one of: ``per-request-header``, ``m2m-client-credentials``,
``static-token``, ``oauth2-passthrough``, or ``no-auth``.
"""
from litellm.types.mcp import MCPAuth
has_server_specific = bool(
mcp_server_auth_headers
and (
mcp_server_auth_headers.get(server.alias or "")
or mcp_server_auth_headers.get(server.server_name or "")
)
)
if has_server_specific or mcp_auth_header:
return "per-request-header"
if server.has_client_credentials:
return "m2m-client-credentials"
if server.authentication_token:
return "static-token"
if oauth2_headers and server.auth_type == MCPAuth.oauth2:
return "oauth2-passthrough"
return "no-auth"
@staticmethod
def build_debug_headers(
*,
inbound_headers: Dict[str, str],
oauth2_headers: Optional[Dict[str, str]],
litellm_api_key: Optional[str],
auth_resolution: str,
server_url: Optional[str],
server_auth_type: Optional[str],
) -> Dict[str, str]:
"""
Build masked debug response headers.
Parameters
----------
inbound_headers : dict
Raw headers received from the MCP client.
oauth2_headers : dict or None
Extracted OAuth2 headers (``{"Authorization": "Bearer ..."}``).
litellm_api_key : str or None
The LiteLLM API key extracted from ``x-litellm-api-key`` or
``Authorization`` header.
auth_resolution : str
Which auth priority was selected for the outbound call.
server_url : str or None
Upstream MCP server URL.
server_auth_type : str or None
The ``auth_type`` configured on the server (e.g. ``oauth2``).
Returns
-------
dict
Headers to include in the response (all values masked).
"""
debug: Dict[str, str] = {}
# --- Inbound auth summary ---
inbound_parts = []
for hdr_name in ("x-litellm-api-key", "authorization", "x-mcp-auth"):
for k, v in inbound_headers.items():
if k.lower() == hdr_name:
inbound_parts.append(f"{hdr_name}={MCPDebug._mask(v)}")
break
debug[f"{_RESPONSE_HEADER_PREFIX}-inbound-auth"] = (
"; ".join(inbound_parts) if inbound_parts else "(none)"
)
# --- OAuth2 token ---
oauth2_token = (oauth2_headers or {}).get("Authorization")
if oauth2_token and litellm_api_key:
oauth2_raw = oauth2_token.removeprefix("Bearer ").strip()
litellm_raw = litellm_api_key.removeprefix("Bearer ").strip()
if oauth2_raw == litellm_raw:
debug[f"{_RESPONSE_HEADER_PREFIX}-oauth2-token"] = (
f"{MCPDebug._mask(oauth2_token)} "
f"(SAME_AS_LITELLM_KEY - likely misconfigured)"
)
else:
debug[f"{_RESPONSE_HEADER_PREFIX}-oauth2-token"] = MCPDebug._mask(
oauth2_token
)
else:
debug[f"{_RESPONSE_HEADER_PREFIX}-oauth2-token"] = MCPDebug._mask(
oauth2_token
)
# --- Auth resolution ---
debug[f"{_RESPONSE_HEADER_PREFIX}-auth-resolution"] = auth_resolution
# --- Server info ---
debug[f"{_RESPONSE_HEADER_PREFIX}-outbound-url"] = server_url or "(unknown)"
debug[f"{_RESPONSE_HEADER_PREFIX}-server-auth-type"] = (
server_auth_type or "(none)"
)
return debug
@staticmethod
def wrap_send_with_debug_headers(
send: Send, debug_headers: Dict[str, str]
) -> Send:
"""
Return a new ASGI ``send`` callable that injects *debug_headers*
into the ``http.response.start`` message.
"""
async def _send_with_debug(message: Message) -> None:
if message["type"] == "http.response.start":
headers = list(message.get("headers", []))
for k, v in debug_headers.items():
headers.append((k.encode(), v.encode()))
message = {**message, "headers": headers}
await send(message)
return _send_with_debug
@staticmethod
def maybe_build_debug_headers(
*,
raw_headers: Optional[Dict[str, str]],
scope: Dict,
mcp_servers: Optional[List[str]],
mcp_auth_header: Optional[str],
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]],
oauth2_headers: Optional[Dict[str, str]],
client_ip: Optional[str],
) -> Dict[str, str]:
"""
Build debug headers if debug mode is enabled, otherwise return empty dict.
This is the single entry point called from the MCP request handler.
"""
if not raw_headers or not MCPDebug.is_debug_enabled(raw_headers):
return {}
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
server_url: Optional[str] = None
server_auth_type: Optional[str] = None
auth_resolution = "no-auth"
for server_name in mcp_servers or []:
server = global_mcp_server_manager.get_mcp_server_by_name(
server_name, client_ip=client_ip
)
if server:
server_url = server.url
server_auth_type = server.auth_type
auth_resolution = MCPDebug.resolve_auth_resolution(
server, mcp_auth_header, mcp_server_auth_headers, oauth2_headers
)
break
scope_headers = MCPRequestHandler._safe_get_headers_from_scope(scope)
litellm_key = MCPRequestHandler.get_litellm_api_key_from_headers(
scope_headers
)
return MCPDebug.build_debug_headers(
inbound_headers=raw_headers,
oauth2_headers=oauth2_headers,
litellm_api_key=litellm_key,
auth_resolution=auth_resolution,
server_url=server_url,
server_auth_type=server_auth_type,
)

Some files were not shown because too many files have changed in this diff Show more