scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

Merge pull request #20738 from BerriAI/main

Browse source 
merge main
This commit is contained in:
Sameer Kankute 2026-02-09 15:07:39 +05:30 • committed by GitHub
commit 2f33445054
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
954 changed files with 30929 additions and 5199 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
.circleci/config.yml
View file

@ -1372,6 +1372,51 @@ jobs:
paths:
- mcp_coverage.xml
- mcp_coverage
agent_testing:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.11.0"
pip install "a2a-sdk"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml agent_coverage.xml
mv .coverage agent_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- agent_coverage.xml
- agent_coverage
guardrails_testing:
docker:
- image: cimg/python:3.11
@ -4264,6 +4309,12 @@ workflows:
only:
- main
- /litellm_.*/
- agent_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- guardrails_testing:
filters:
branches:
@ -4371,6 +4422,7 @@ workflows:
- llm_translation_testing
- realtime_translation_testing
- mcp_testing
- agent_testing
- google_generate_content_endpoint_testing
- guardrails_testing
- llm_responses_api_testing
@ -4449,6 +4501,7 @@ workflows:
- llm_translation_testing
- realtime_translation_testing
- mcp_testing
- agent_testing
- google_generate_content_endpoint_testing
- llm_responses_api_testing
- ocr_testing

1
CLAUDE.md
View file

@ -90,6 +90,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- Pydantic v2 for data validation
- Async/await patterns throughout
- Type hints required for all public APIs
- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
### Testing Strategy
- Unit tests in `tests/test_litellm/`

48
CONTRIBUTING.md
View file

@ -7,11 +7,20 @@ Thank you for your interest in contributing to LiteLLM! We welcome contributions
Here are the core requirements for any PR submitted to LiteLLM:
- [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
#### Proxy (Backend) PRs
- [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
- [ ] **Ensure your PR passes all checks**:
- [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
- [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
#### UI PRs
- [ ] **Ensure the UI builds successfully** - `npm run build`
- [ ] **Ensure all UI unit tests pass** - `npm run test`
- [ ] **Add tests for new components or logic** - If you are adding a new component or new logic, add corresponding tests
## **Contributor License Agreement (CLA)**
@ -245,6 +254,43 @@ docker run \
--config /app/config.yaml --detailed_debug
```
## UI Development
### 1. Setup Your Local UI Development Environment
```bash
# Clone the repo (if you haven't already)
git clone https://github.com/YOUR_USERNAME/litellm.git
cd litellm
# Navigate to the UI dashboard directory
cd ui/litellm-dashboard
# Install dependencies
npm install
# Start the development server
npm run dev
```
### 2. Adding UI Tests
If you are adding a **new component** or **new logic**, you must add corresponding tests.
### 3. Running UI Unit Tests
```bash
npm run test
```
### 4. Building the UI
Ensure the UI builds successfully before submitting your PR:
```bash
npm run build
```
## Submitting Your PR
1. **Push your branch**: `git push origin your-feature-branch`

5
Dockerfile
View file

@ -3,6 +3,7 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
@ -62,6 +63,10 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi
# Remove test files and keys from dependencies
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete

65
Makefile
View file

@ -1,7 +1,10 @@
# LiteLLM Makefile
# Simple Makefile for running tests and basic development tasks
.PHONY: help test test-unit test-integration test-unit-helm lint format install-dev install-proxy-dev install-test-deps install-helm-unittest check-circular-imports check-import-safety
.PHONY: help test test-unit test-integration test-unit-helm \
info lint lint-dev format \
install-dev install-proxy-dev install-test-deps \
install-helm-unittest check-circular-imports check-import-safety
# Default target
help:
@ -25,6 +28,13 @@ help:
@echo " make test-integration - Run integration tests"
@echo " make test-unit-helm - Run helm unit tests"
# Keep PIP simple for edge cases:
PIP := $(shell command -v pip > /dev/null 2>&1 && echo "pip" || echo "python3 -m pip")
# Show info
info:
@echo "PIP: $(PIP)"
# Installation targets
install-dev:
poetry install --with dev
@ -34,19 +44,19 @@ install-proxy-dev:
# CI-compatible installations (matches GitHub workflows exactly)
install-dev-ci:
pip install openai==2.8.0
$(PIP) install openai==2.8.0
poetry install --with dev
pip install openai==2.8.0
$(PIP) install openai==2.8.0
install-proxy-dev-ci:
poetry install --with dev,proxy-dev --extras proxy
pip install openai==2.8.0
$(PIP) install openai==2.8.0
install-test-deps: install-proxy-dev
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
poetry run pip install openapi-core
cd enterprise && poetry run pip install -e . && cd ..
poetry run $(PIP) install "pytest-retry==1.6.3"
poetry run $(PIP) install pytest-xdist
poetry run $(PIP) install openapi-core
cd enterprise && poetry run $(PIP) install -e . && cd ..
install-helm-unittest:
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
@ -62,8 +72,40 @@ format-check: install-dev
lint-ruff: install-dev
cd litellm && poetry run ruff check . && cd ..
# faster linter for developing ...
# inspiration from:
# https://github.com/astral-sh/ruff/discussions/10977
# https://github.com/astral-sh/ruff/discussions/4049
lint-format-changed: install-dev
@git diff origin/main --unified=0 --no-color -- '*.py' | \
perl -ne '\
if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
print "$$file:$$start:1-$$end:999\n"; \
}' | \
while read range; do \
file="$${range%%:*}"; \
lines="$${range#*:}"; \
echo "Formatting $$file (lines $$lines)"; \
poetry run ruff format --range "$$lines" "$$file"; \
done
lint-ruff-dev: install-dev
@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
cd litellm && \
(poetry run ruff check . --output-format=pylint || true) > "$$tmpfile" && \
poetry run diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
cd .. ; \
rm -f "$$tmpfile"
lint-ruff-FULL-dev: install-dev
@files=$$(git diff --name-only origin/main -- '*.py'); \
if [ -n "$$files" ]; then echo "$$files" | xargs poetry run ruff check; \
else echo "No changed .py files to check."; fi
lint-mypy: install-dev
poetry run pip install types-requests types-setuptools types-redis types-PyYAML
poetry run $(PIP) install types-requests types-setuptools types-redis types-PyYAML
cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
lint-black: format-check
@ -72,11 +114,14 @@ check-circular-imports: install-dev
cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
check-import-safety: install-dev
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
@poetry run python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Combined linting (matches test-linting.yml workflow)
lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
# Testing targets
test:
poetry run pytest tests/

2
README.md
View file

@ -309,7 +309,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
| [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ | | | ✅ | | | | |
| [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | | | | | | | |
| [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | | | | | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | | ✅ | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |

4
ci_cd/security_scans.sh
View file

@ -155,6 +155,10 @@ run_grype_scans() {
"CVE-2025-12781" # No fix available yet
"CVE-2025-11468" # No fix available yet
"CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
"GHSA-7h2j-956f-4vf2" # @isaacs/brace-expansion ReDoS - npm tooling dependency, not used in application runtime
"GHSA-hx9q-6w63-j58v" # orjson deep recursion - no fix available yet
"GHSA-8qq5-rm4j-mr97" # node-tar symlink poisoning - npm tooling dependency, tar CLI not exposed in application code
"GHSA-29xp-372q-xqph" # node-tar race condition - npm tooling dependency, tar CLI not exposed in application code
)
# Build JSON array of allowlisted CVE IDs for jq

4
docker/Dockerfile.database
View file

@ -64,6 +64,10 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi
# Install semantic_router and aurelio-sdk using script
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh

5
docker/Dockerfile.non_root
View file

@ -47,7 +47,6 @@ RUN mkdir -p /var/lib/litellm/ui && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
rm -f package-lock.json && \
npm install --legacy-peer-deps && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
@ -147,6 +146,10 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
fi; \
fi
# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi
# Permissions, cleanup, and Prisma prep
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && \

403
docs/my-website/blog/claude_opus_4_6/index.md Normal file
View file

@ -0,0 +1,403 @@
---
slug: claude_opus_4_6
title: "Day 0 Support: Claude Opus 4.6"
date: 2026-02-05T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
description: "Day 0 support for Claude Opus 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
tags: [anthropic, claude, opus 4.6]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM now supports Claude Opus 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
## Docker Image
```bash
docker pull ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6
```
## Usage - Anthropic
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-opus-4-6
litellm_params:
model: anthropic/claude-opus-4-6
api_key: os.environ/ANTHROPIC_API_KEY
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>
## Usage - Azure
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-opus-4-6
litellm_params:
model: azure_ai/claude-opus-4-6
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE # https://<resource>.services.ai.azure.com
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
-e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>
## Usage - Vertex AI
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-opus-4-6
litellm_params:
model: vertex_ai/claude-opus-4-6
vertex_project: os.environ/VERTEX_PROJECT
vertex_location: us-east5
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e VERTEX_PROJECT=$VERTEX_PROJECT \
-e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
-v $(pwd)/config.yaml:/app/config.yaml \
-v $(pwd)/credentials.json:/app/credentials.json \
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>
## Usage - Bedrock
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-opus-4-6
litellm_params:
model: bedrock/anthropic.claude-opus-4-6-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>
## Compaction
Litellm supports enabling compaction for the new claude-opus-4-6.
### Enabling Compaction
To enable compaction, add the `context_management` parameter with the `compact_20260112` edit type:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "What is the weather in San Francisco?"
}
],
"context_management": {
"edits": [
{
"type": "compact_20260112"
}
]
},
"max_tokens": 100
}'
```
All the parameters supported for context_management by anthropic are supported and can be directly added. Litellm automatically adds the `compact-2026-01-12` beta header in the request.
### Response with Compaction Block
The response will include the compaction summary in `provider_specific_fields.compaction_blocks`:
```json
{
"id": "chatcmpl-a6c105a3-4b25-419e-9551-c800633b6cb2",
"created": 1770357619,
"model": "claude-opus-4-6",
"object": "chat.completion",
"choices": [
{
"finish_reason": "length",
"index": 0,
"message": {
"content": "I don't have access to real-time data, so I can't provide the current weather in San Francisco. To get up-to-date weather information, I'd recommend checking:\n\n- **Weather websites** like weather.com, accuweather.com, or wunderground.com\n- **Search engines** – just Google \"San Francisco weather\"\n- **Weather apps** on your phone (e.g., Apple Weather, Google Weather)\n- **National",
"role": "assistant",
"provider_specific_fields": {
"compaction_blocks": [
{
"type": "compaction",
"content": "Summary of the conversation: The user requested help building a web scraper..."
}
]
}
}
}
],
"usage": {
"completion_tokens": 100,
"prompt_tokens": 86,
"total_tokens": 186
}
}
```
### Using Compaction Blocks in Follow-up Requests
To continue the conversation with compaction, include the compaction block in the assistant message's `provider_specific_fields`:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "How can I build a web scraper?"
},
{
"role": "assistant",
"content": [
{
"type": "text",
"text": "Certainly! To build a basic web scraper, you'll typically use a programming language like Python along with libraries such as `requests` (for fetching web pages) and `BeautifulSoup` (for parsing HTML). Here's a basic example:\n\n```python\nimport requests\nfrom bs4 import BeautifulSoup\n\nurl = 'https://example.com'\nresponse = requests.get(url)\nsoup = BeautifulSoup(response.text, 'html.parser')\n\n# Extract and print all text\ntext = soup.get_text()\nprint(text)\n```\n\nLet me know what you're interested in scraping or if you need help with a specific website!"
}
],
"provider_specific_fields": {
"compaction_blocks": [
{
"type": "compaction",
"content": "Summary of the conversation: The user asked how to build a web scraper, and the assistant gave an overview using Python with requests and BeautifulSoup."
}
]
}
},
{
"role": "user",
"content": "How do I use it to scrape product prices?"
}
],
"context_management": {
"edits": [
{
"type": "compact_20260112"
}
]
},
"max_tokens": 100
}'
```
### Streaming Support
Compaction blocks are also supported in streaming mode. You'll receive:
- `compaction_start` event when a compaction block begins
- `compaction_delta` events with the compaction content
- The accumulated `compaction_blocks` in `provider_specific_fields`
## Adaptive Thinking
LiteLLM supports adaptive thinking through the `reasoning_effort` parameter:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "Solve this complex problem: What is the optimal strategy for..."
}
],
"reasoning_effort": "high"
}'
```
## Effort Levels
Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "Explain quantum computing"
}
],
"output_config": {
"effort": "medium"
}
}'
```
You can use reasoning effort plus output_config to have more control on the model.
## 1M Token Context (Beta)
Opus 4.6 supports 1M token context. Premium pricing applies for prompts exceeding 200k tokens ($10/$37.50 per million input/output tokens). LiteLLM supports cost calculations for 1M token contexts.
## US-Only Inference
Available at 1.1× token pricing. LiteLLM supports this pricing model.

220
docs/my-website/blog/fastapi_middleware_performance/index.mdx Normal file
View file

@ -0,0 +1,220 @@
---
slug: fastapi-middleware-performance
title: "Your Middleware Could Be a Bottleneck"
date: 2026-02-07T10:00:00
authors:
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Ryan Crabbe
title: "Performance Engineer, LiteLLM"
url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
description: "How we improved LiteLLM proxy latency and throughput by replacing a single middleware base class"
tags: [performance, fastapi, middleware]
hide_table_of_contents: false
---
import { BaseHTTPMiddlewareAnimation, PureASGIAnimation, BenchmarkVisualization } from '@site/src/components/MiddlewareDiagrams';
> How we improved LiteLLM proxy latency and throughput by replacing a single, simple middleware base class
---
## Our Setup
The LiteLLM proxy server has two middleware layers. The first is Starlette's `CORSMiddleware` (re-exported by FastAPI), which is a pure ASGI middleware. Then we have a simple BaseHTTPMiddleware called PrometheusAuthMiddleware.
The job of `PrometheusAuthMiddleware` is to authenticate requests to the `/metrics` endpoint. It's not on by default, you enable it with a flag in your proxy config:
<details>
<summary>Proxy config flag</summary>
```yaml
litellm_settings:
require_auth_for_metrics_endpoint: true
```
</details>
The middleware checks two things: is the request hitting `/metrics`, and is auth even enabled? If both checks fail, which they do for the vast majority of requests, it just passes the request through unchanged.
<details>
<summary>PrometheusAuthMiddleware source</summary>
```python
class PrometheusAuthMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
if self._is_prometheus_metrics_endpoint(request):
if self._should_run_auth_on_metrics_endpoint() is True:
try:
await user_api_key_auth(request=request, api_key=...)
except Exception as e:
return JSONResponse(status_code=401, content=...)
response = await call_next(request)
return response
@staticmethod
def _is_prometheus_metrics_endpoint(request: Request):
if "/metrics" in request.url.path:
return True
return False
```
</details>
Looks harmless. Subclass `BaseHTTPMiddleware`, implement `dispatch()`, done. This is what you will see in Starlette's documentation<sup>[1](#footnote-1)</sup>.
{/* truncate */}
---
## What BaseHTTPMiddleware Actually Does
When you write a `dispatch()` method, you'd expect the request to flow straight through your function and out the other side. What actually happens is much more involved.
On every request, even a pure passthrough (meaning nothing happens), `BaseHTTPMiddleware` creates **7 intermediate objects and tasks**:
<BaseHTTPMiddlewareAnimation />
It wraps the request in a new object to track body state, creates a synchronization event, allocates an in-memory channel to pass messages between your middleware and the inner app, sets up a task group to manage the lifecycle, and then runs your actual route handler in a *separate background task* when you call `call_next()`. The response body then flows back through that in-memory channel, gets re-wrapped in a streaming response object, and finally reaches the caller. That's a lot.
For a middleware that for us, does nothing on 99.9% of requests, paying this cost doesn't make sense.
Compare that to a pure ASGI middleware, which we can have just check the request path and continue along.
<PureASGIAnimation />
Our middleware is doing something really simple. For the vast majority of requests it doesn't need to do anything at all but just let the request pass through. It doesn't need task groups, memory streams, or cancel scopes. It needs a function call.
---
## Comparing Both
We replaced the `BaseHTTPMiddleware` subclass with a pure ASGI middleware. To benchmark the difference, we used Apache Bench<sup>[2](#footnote-2)</sup> to compare both configurations of LiteLLM's middleware stack: the old setup (1 pure ASGI + 1 `BaseHTTPMiddleware`) against the new setup (2 pure ASGI).
A minimal FastAPI app serves `GET /health` → `PlainTextResponse("ok")`. The endpoint does zero work to isolate the middleware overhead: any difference between configs is purely the cost of the middleware plumbing itself. Both middlewares are just calling the next layer. Same work, different base class.
Apache Bench (`ab`) fires requests at the server with 1,000 concurrent connections and a single uvicorn worker. One worker means one event loop, so the benchmark directly measures how each middleware design handles concurrent load on a single thread.
<BenchmarkVisualization />
<details>
<summary>Try it yourself</summary>
Save the script below as `benchmark_middleware.py`, then run:
```bash
# Terminal 1 — start the "before" server (1 ASGI + 1 BaseHTTPMiddleware)
python benchmark_middleware.py --middleware mixed
# Terminal 2 — benchmark it
ab -n 50000 -c 1000 http://localhost:8000/health
# Stop the server, then start the "after" server (2x pure ASGI)
python benchmark_middleware.py --middleware asgi
# Terminal 2 — benchmark again
ab -n 50000 -c 1000 http://localhost:8000/health
```
```python
import argparse
import uvicorn
from fastapi import FastAPI
from fastapi.responses import PlainTextResponse
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from starlette.types import ASGIApp, Receive, Scope, Send
class NoOpBaseHTTPMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
return await call_next(request)
class NoOpPureASGIMiddleware:
def __init__(self, app: ASGIApp) -> None:
self.app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
await self.app(scope, receive, send)
def create_app(middleware_type: str | None = None, layers: int = 2) -> FastAPI:
app = FastAPI()
@app.get("/health")
async def health():
return PlainTextResponse("ok")
if middleware_type == "mixed":
app.add_middleware(NoOpBaseHTTPMiddleware)
app.add_middleware(NoOpPureASGIMiddleware)
elif middleware_type == "asgi":
for _ in range(layers):
app.add_middleware(NoOpPureASGIMiddleware)
return app
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--middleware", choices=["asgi", "mixed"], default=None)
parser.add_argument("--layers", type=int, default=2)
parser.add_argument("--port", type=int, default=8000)
args = parser.parse_args()
app = create_app(middleware_type=args.middleware, layers=args.layers)
uvicorn.run(app, host="0.0.0.0", port=args.port, workers=1, log_level="warning")
```
</details>
---
## Our Change
Here's what we replaced it with:
```python
class PrometheusAuthMiddleware:
def __init__(self, app: ASGIApp) -> None:
self.app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http" or "/metrics" not in scope.get("path", ""):
await self.app(scope, receive, send)
return
if litellm.require_auth_for_metrics_endpoint is True:
request = Request(scope, receive)
api_key = request.headers.get("Authorization") or ""
try:
await user_api_key_auth(request=request, api_key=api_key)
except Exception as e:
# send 401 directly via ASGI protocol
...
return
await self.app(scope, receive, send)
```
For the 99.9% of requests that aren't hitting `/metrics`, the middleware is now one dict lookup, one string check, and one function call. No objects allocated, no tasks spawned.
It's important to evaluate if the tools you're using are the right fit for the job as your software grows and handles more responsiblity. We're now putting in a static analysis check to prevent this from happening again with any newly introduced middlewares. If we find the use case is necessary then that's okay and we'll reevalute but for everything LiteLLM needs to do at the moment it's not.
This middleware change was one part of a broader optimization effort on the LiteLLM proxy. Across all optimizations combined, we've measured about a **30% reduction in proxy overhead** over the past two weeks.
---
<a id="footnote-1"></a>
<sup>1</sup> [Starlette Middleware — BaseHTTPMiddleware](https://starlette.dev/middleware/#basehttpmiddleware)
<a id="footnote-2"></a>
<sup>2</sup> [Apache HTTP server benchmarking tool (`ab`)](https://httpd.apache.org/docs/2.4/programs/ab.html)

136
docs/my-website/blog/litellm_observatory/index.md Normal file
View file

@ -0,0 +1,136 @@
---
slug: litellm-observatory
title: "Improve release stability with 24 hour load tests"
date: 2026-02-06T10:00:00
authors:
- name: Alexsander Hamir
title: "Performance Engineer, LiteLLM"
url: https://www.linkedin.com/in/alexsander-baptista/
image_url: https://github.com/AlexsanderHamir.png
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "How we built a long-running, release-validation system to catch regressions before they reach users."
tags: [testing, observability, reliability, releases]
hide_table_of_contents: false
---
![LiteLLM Observatory](https://raw.githubusercontent.com/AlexsanderHamir/assets/main/Screenshot%202026-01-31%20175355.png)
# Improve release stability with 24 hour load tests
As LiteLLM adoption has grown, so have expectations around reliability, performance, and operational safety. Meeting those expectations requires more than correctness-focused tests, it requires validating how the system behaves over time, under real-world conditions.
This post introduces **LiteLLM Observatory**, a long-running release-validation system we built to catch regressions before they reach users.
---
## Why We Built the Observatory
LiteLLM operates at the intersection of external providers, long-lived network connections, and high-throughput workloads. While our unit and integration tests do an excellent job validating correctness, they are not designed to surface issues that only appear after extended operation.
A subtle lifecycle edge case discovered in v1.81.3 reinforced the need for stronger release validation in this area.
---
## A Real-World Lifecycle Edge Case
In v1.81.3, we shipped a fix for an HTTP client memory leak. The change passed unit and integration tests and behaved correctly in short-lived runs.
The issue that surfaced was not caused by a single incorrect line of logic, but by how multiple components interacted over time:
- A cached `httpx` client was configured with a 1-hour TTL
- When the cache expired, the underlying HTTP connection was closed as expected
- A higher-level client continued to hold a reference to that connection
- Subsequent requests failed with:
```
Cannot send a request, as the client has been closed
```
**Before (with bug):**
| Provider | Requests | Success | Failures | Fail % |
|----------|----------|---------|----------|--------|
| OpenAI | 720,000 | 432,000 | 288,000 | 40% |
| Azure | 692,000 | 415,200 | 276,800 | 40% |
**After (fixed):**
| Provider | Requests | Success | Failures | Fail % |
|----------|------------|-----------|----------|---------|
| OpenAI | 1,200,000 | 1,199,988 | 12 | 0.001% |
| Azure | 1,150,000 | 1,149,982 | 18 | 0.002% |
Our focus moving forward is on being the first to detect issues, even when they aren’t covered by unit tests. LiteLLM Observatory is designed to surface latency regressions, OOMs, and failure modes that only appear under real traffic patterns in **our own production deployments** during release validation.
---
### How the Observatory Works
[LiteLLM Observatory](https://github.com/BerriAI/litellm-observatory) is a testing service that runs long-running tests against our LiteLLM deployments. We trigger tests by sending API requests, and results are automatically sent to Slack when tests complete.
#### How Tests Run
1. **Start a Test**: We send a request to the Observatory API with:
- Which LiteLLM deployment to test (URL and API key)
- Which test to run (e.g., `TestOAIAzureRelease`)
- Test settings (which models to test, how long to run, failure thresholds)
2. **Smart Queueing**:
- The system checks whether we are attempting to run the exact same test more than once
- If a duplicate test is already running or queued, we receive an error to avoid wasting resources
- Otherwise, the test is added to a queue and runs when capacity is available (up to 5 tests can run concurrently by default)
3. **Instant Response**: The API responds immediately—we do not wait for the test to finish. Tests may run for hours, but the request itself completes in milliseconds.
4. **Background Execution**:
- The test runs in the background, issuing requests against our LiteLLM deployment
- It tracks request success and failure rates over time
- When the test completes, results are automatically posted to our Slack channel
#### Example: The OpenAI / Azure Reliability Test
The `TestOAIAzureRelease` test is designed to catch a class of bugs that only surface after sustained runtime:
- **Duration**: Runs continuously for 3 hours
- **Behavior**: Cycles through specified models (such as `gpt-4` and `gpt-3.5-turbo`), issuing requests continuously
- **Why 3 Hours**: This helps catch issues where HTTP clients degrade or fail after extended use (for example, a bug observed in LiteLLM v1.81.3)
- **Pass / Fail Criteria**: The test passes if fewer than 1% of requests fail. If the failure rate exceeds 1%, the test fails and we are notified in Slack
- **Key Detail**: The same HTTP client is reused for the entire run, allowing us to detect lifecycle-related bugs that only appear under prolonged reuse
#### When We Use It
- **Before Deployments**: Run tests before promoting a new LiteLLM version to production
- **Routine Validation**: Schedule regular runs (daily or weekly) to catch regressions early
- **Issue Investigation**: Run tests on demand when we suspect a deployment issue
- **Long-Running Failure Detection**: Identify bugs that only appear under sustained load, beyond what short smoke tests can reveal
### Complementing Unit Tests
Unit tests remain a foundational part of our development process. They are fast and precise, but they don’t cover:
- Real provider behavior
- Long-lived network interactions
- Resource lifecycle edge cases
- Time-dependent regressions
LiteLLM Observatory complements unit tests by validating the system as it actually runs in production-like environments.
---
### Looking Ahead
Reliability is an ongoing investment.
LiteLLM Observatory is one of several systems we’re building to continuously raise the bar on release quality and operational safety. As LiteLLM evolves, so will our validation tooling, informed by real-world usage and lessons learned.
We’ll continue to share those improvements openly as we go.

19
docs/my-website/docs/completion/web_search.md
View file

@ -18,12 +18,29 @@ Each provider uses their own search backend:
| Provider | Search Engine | Notes |
|----------|---------------|-------|
| **OpenAI** (`gpt-4o-search-preview`) | OpenAI's internal search | Real-time web data |
| **OpenAI** (`gpt-4o-search-preview`, `gpt-4o-mini-search-preview`, `gpt-5-search-api`) | OpenAI's internal search | Real-time web data |
| **xAI** (`grok-3`) | xAI's search + X/Twitter | Real-time social media data |
| **Google AI/Vertex** (`gemini-2.0-flash`) | **Google Search** | Uses actual Google search results |
| **Anthropic** (`claude-3-5-sonnet`) | Anthropic's web search | Real-time web data |
| **Perplexity** | Perplexity's search engine | AI-powered search and reasoning |
:::warning Important: Only Search Models Support `web_search_options`
For OpenAI, only dedicated search models support the `web_search_options` parameter:
- `gpt-4o-search-preview`
- `gpt-4o-mini-search-preview`
- `gpt-5-search-api`
**Regular models like `gpt-5`, `gpt-4.1`, `gpt-4o` do not support `web_search_options`**
:::
:::tip The `web_search_options` parameter is optional
Search models (like `gpt-4o-search-preview`) **automatically search the web** even without the `web_search_options` parameter.
Use `web_search_options` when you need to:
- Adjust `search_context_size` (`"low"`, `"medium"`, `"high"`)
- Specify `user_location` for localized results
:::
:::info
**Anthropic Web Search Models**: Claude models that support web search: `claude-3-5-sonnet-latest`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-latest`, `claude-3-5-haiku-20241022`, `claude-3-7-sonnet-20250219`
:::

150
docs/my-website/docs/extras/contributing_code.md
View file

@ -1,27 +1,36 @@
# Contributing Code
## **Checklist before submitting a PR**
## Checklist before submitting a PR
Here are the core requirements for any PR submitted to LiteLLM
Here are the core requirements for any PR submitted to LiteLLM:
- [ ] Sign the Contributor License Agreement (CLA) - [see details](#contributor-license-agreement-cla)
- [ ] Add testing, **Adding at least 1 test is a hard requirement** - [see details](#2-adding-testing-to-your-pr)
- [ ] Ensure your PR passes the following tests:
- [ ] [Unit Tests](#3-running-unit-tests)
- [ ] [Formatting / Linting Tests](#35-running-linting-tests)
- [ ] Keep scope as isolated as possible. As a general rule, your changes should address 1 specific problem at a time
- [ ] Sign the [Contributor License Agreement (CLA)](#contributor-license-agreement-cla)
- [ ] Keep scope as isolated as possible — your changes should address **one specific problem** at a time
## **Contributor License Agreement (CLA)**
### Proxy (Backend) PRs
- [ ] Add testing — **at least 1 test is a hard requirement** ([details](#2-adding-tests))
- [ ] Ensure your PR passes:
- [ ] [Unit Tests](#3-running-unit-tests) — `make test-unit`
- [ ] [Formatting / Linting Tests](#4-running-linting-tests) — `make lint`
### UI PRs
- [ ] Ensure the UI builds successfully — `npm run build`
- [ ] Ensure all UI unit tests pass — `npm run test`
- [ ] If you are adding a **new component** or **new logic**, add corresponding tests
## Contributor License Agreement (CLA)
Before contributing code to LiteLLM, you must sign our [Contributor License Agreement (CLA)](https://cla-assistant.io/BerriAI/litellm). This is a legal requirement for all contributions to be merged into the main repository. The CLA helps protect both you and the project by clearly defining the terms under which your contributions are made.
**Important:** We strongly recommend reviewing and signing the CLA before starting work on your contribution to avoid any delays in the PR process. You can find the CLA [here](https://cla-assistant.io/BerriAI/litellm) and sign it through our CLA management system when you submit your first PR.
**Important:** We strongly recommend signing the CLA **before** starting work on your contribution to avoid delays in the review process. You can find and sign the CLA [here](https://cla-assistant.io/BerriAI/litellm).
## Quick start
---
## 1. Setup your local dev environment
## Proxy (Backend)
Here's how to modify the repo locally:
### 1. Setting up your local dev environment
Step 1: Clone the repo
@ -29,56 +38,53 @@ Step 1: Clone the repo
git clone https://github.com/BerriAI/litellm.git
```
Step 2: Install dev dependencies:
Step 2: Install dev dependencies
```shell
poetry install --with dev --extras proxy
```
That's it, your local dev environment is ready!
### 2. Adding tests
## 2. Adding Testing to your PR
- Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm).
- This directory mirrors the `litellm/` directory 1:1 and should **only** contain mocked tests.
- **Do not** add real LLM API calls to this directory.
- Add your test to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm)
#### File naming convention for `tests/test_litellm/`
- This directory 1:1 maps the the `litellm/` directory, and can only contain mocked tests.
- Do not add real llm api calls to this directory.
The test directory follows the same structure as `litellm/`:
### 2.1 File Naming Convention for `tests/test_litellm/`
The `tests/test_litellm/` directory follows the same directory structure as `litellm/`.
- `litellm/proxy/test_caching_routes.py` maps to `litellm/proxy/caching_routes.py`
- `test_{filename}.py` maps to `litellm/{filename}.py`
- `litellm/proxy/test_caching_routes.py` maps to `litellm/proxy/caching_routes.py`
## 3. Running Unit Tests
### 3. Running unit tests
run the following command on the root of the litellm directory
Run the following command from the root of the `litellm` directory:
```shell
make test-unit
```
## 3.5 Running Linting Tests
### 4. Running linting tests
run the following command on the root of the litellm directory
Run the following command from the root of the `litellm` directory:
```shell
make lint
```
LiteLLM uses mypy for linting. On ci/cd we also run `black` for formatting.
LiteLLM uses `mypy` for type checking. CI/CD also runs `black` for formatting.
## 4. Submit a PR with your changes!
### 5. Submit a PR
- push your fork to your GitHub repo
- submit a PR from there
- Push your changes to your fork on GitHub
- Open a Pull Request from your fork
## Advanced
---
### Building LiteLLM Docker Image
## UI
Some people might want to build the LiteLLM docker image themselves. Follow these instructions if you want to build / run the LiteLLM Docker Image yourself.
### 1. Setting up your local dev environment
Step 1: Clone the repo
@ -86,17 +92,72 @@ Step 1: Clone the repo
git clone https://github.com/BerriAI/litellm.git
```
Step 2: Build the Docker Image
Step 2: Navigate to the UI dashboard directory
Build using Dockerfile.non_root
```shell
cd ui/litellm-dashboard
```
Step 3: Install dependencies
```shell
npm install
```
Step 4: Start the development server
```shell
npm run dev
```
### 2. Adding tests
If you are adding a **new component** or **new logic**, you must add corresponding tests.
### 3. Running UI unit tests
```shell
npm run test
```
### 4. Building the UI
Ensure the UI builds successfully before submitting your PR:
```shell
npm run build
```
### 5. Submit a PR
- Push your changes to your fork on GitHub
- Open a Pull Request from your fork
---
## Advanced
### Building the LiteLLM Docker Image
Follow these instructions if you want to build and run the LiteLLM Docker image yourself.
Step 1: Clone the repo
```shell
git clone https://github.com/BerriAI/litellm.git
```
Step 2: Build the Docker image
Build using `Dockerfile.non_root`:
```shell
docker build -f docker/Dockerfile.non_root -t litellm_test_image .
```
Step 3: Run the Docker Image
Step 3: Run the Docker image
Make sure config.yaml is present in the root directory. This is your litellm proxy config file.
Make sure `config.yaml` is present in the root directory. This is your LiteLLM proxy config file.
```shell
docker run \
@ -107,18 +168,19 @@ docker run \
litellm_test_image \
--config /app/config.yaml --detailed_debug
```
### Running LiteLLM Proxy Locally
1. cd into the `proxy/` directory
### Running the LiteLLM Proxy Locally
```
1. Navigate to the `proxy/` directory:
```shell
cd litellm/litellm/proxy
```
2. Run the proxy
2. Run the proxy:
```shell
python3 proxy_cli.py --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
```

251
docs/my-website/docs/mcp_public_internet.md Normal file
View file

@ -0,0 +1,251 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Exposing MCPs on the Public Internet
Control which MCP servers are visible to external callers (e.g., ChatGPT, Claude Desktop) vs. internal-only callers. This is useful when you want a subset of your MCP servers available publicly while keeping sensitive servers restricted to your private network.
## Overview
| Property | Details |
|-------|-------|
| Description | IP-based access control for MCP servers — external callers only see servers marked as public |
| Setting | `available_on_public_internet` on each MCP server |
| Network Config | `mcp_internal_ip_ranges` in `general_settings` |
| Supported Clients | ChatGPT, Claude Desktop, Cursor, OpenAI API, or any MCP client |
## How It Works
When a request arrives at LiteLLM's MCP endpoints, LiteLLM checks the caller's IP address to determine whether they are an **internal** or **external** caller:
1. **Extract the client IP** from the incoming request (supports `X-Forwarded-For` when configured behind a reverse proxy).
2. **Classify the IP** as internal or external by checking it against the configured private IP ranges (defaults to RFC 1918: `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `127.0.0.0/8`).
3. **Filter the server list**:
- **Internal callers** see all MCP servers (public and private).
- **External callers** only see servers with `available_on_public_internet: true`.
This filtering is applied at every MCP access point: the MCP registry, tool listing, tool calling, dynamic server routes, and OAuth discovery endpoints.
```mermaid
flowchart TD
A[Incoming MCP Request] --> B[Extract Client IP Address]
B --> C{Is IP in private ranges?}
C -->|Yes - Internal caller| D[Return ALL MCP servers]
C -->|No - External caller| E[Return ONLY servers with<br/>available_on_public_internet = true]
```
## Walkthrough
This walkthrough covers two flows:
1. **Adding a public MCP server** (DeepWiki) and connecting to it from ChatGPT
2. **Making an existing server private** (Exa) and verifying ChatGPT no longer sees it
### Flow 1: Add a Public MCP Server (DeepWiki)
DeepWiki is a free MCP server — a good candidate to expose publicly so AI gateway users can access it from ChatGPT.
#### Step 1: Create the MCP Server
Navigate to the MCP Servers page and click **"+ Add New MCP Server"**.
![Click Add New MCP Server](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/28cc27c2-d980-4255-b552-ebf542ef95be/ascreenshot_30a7e3c043834f1c87b69e6ffc5bba4f_text_export.jpeg)
The create dialog opens. Enter **"DeepWiki"** as the server name.
![Enter server name](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/8c733c38-310a-40ef-8a5b-7af91cc7f74f/ascreenshot_16df83fed5bd4683a22a042e07063cec_text_export.jpeg)
For the transport type dropdown, select **HTTP** since DeepWiki uses the Streamable HTTP transport.
![Select transport type](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/e473f603-d692-40c7-a218-866c2e1cb554/ascreenshot_e93997971f2f44beac6152786889addf_text_export.jpeg)
Now scroll down to the MCP Server URL field.
![Configure server](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/b08d3c1f-9279-45b6-8efb-f73008901da6/ascreenshot_ce0de66f230a41b0a454e76653429021_text_export.jpeg)
Enter the DeepWiki MCP URL: `https://mcp.deepwiki.com/mcp`.
![Enter MCP server URL](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/e59f8285-cfde-4c57-aa79-24244acc9160/ascreenshot_8d575c66dc614a4183212ba282d22b41_text_export.jpeg)
With the name, transport, and URL filled in, the basic server configuration is complete.
![Server URL configured](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/0f1af7ed-760d-4445-bdec-3da706d4eef4/ascreenshot_d7d6db69bc254ded871d14a71188a212_text_export.jpeg)
#### Step 2: Enable "Available on Public Internet"
Before creating, scroll down and expand the **Permission Management / Access Control** section. This is where you control who can see this server.
![Expand Permission Management](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/cc10dea2-6028-4a27-a33b-1b1b7212efb5/ascreenshot_0fdd152b862a4bf39973bc805ce64c57_text_export.jpeg)
Toggle **"Available on Public Internet"** on. This is the key setting — it tells LiteLLM that external callers (like ChatGPT connecting from the public internet) should be able to discover and use this server.
![Toggle Available on Public Internet](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/39c14543-c5ae-4189-8f85-9efc87135820/ascreenshot_9991f54910c24e21bba5c05ea4fa8e28_text_export.jpeg)
With the toggle enabled, click **"Create"** to save the server.
![Click Create](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/843be209-aade-44f4-98da-e55d1644854c/ascreenshot_8cfc90345a5f4d069b397e80d0a6e449_text_export.jpeg)
#### Step 3: Connect from ChatGPT
Now let's verify it works. Open ChatGPT and look for the MCP server icon to add a new connection. The endpoint to use is `<your-litellm-url>/mcp`.
![ChatGPT add MCP server](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/58b5f674-edf4-4156-a5fa-5fdc8ed5d7b9/ascreenshot_36735f7c37394e919793968794614126_text_export.jpeg)
In the dropdown, select **"Add an MCP server"** to configure a new connection.
![ChatGPT MCP server option](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/f89da8af-bc61-44a7-a765-f52733f4970d/ascreenshot_6410a917b782437eb558de3bfcd35ffd_text_export.jpeg)
ChatGPT asks for a server label. Give it a recognizable name like "LiteLLM".
![Enter server label](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/88505afe-07c1-4674-a89c-8035a5d05eb6/ascreenshot_143aefc38ddd4d3f9f5823ca2cc09bc2_text_export.jpeg)
Next, enter the Server URL. This should be your LiteLLM proxy's MCP endpoint — `<your-litellm-url>/mcp`.
![Enter LiteLLM MCP URL](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/9048be4a-7e40-43e7-9789-059fed2741a6/ascreenshot_e81232c17fd148f48f0ae552e9dc2a10_text_export.jpeg)
Paste your LiteLLM URL and confirm it looks correct.
![URL pasted](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/7707e796-e146-47c8-bce0-58e6f4076272/ascreenshot_0710dc58b8ed4d6887856b1388d59329_text_export.jpeg)
ChatGPT also needs authentication. Enter your LiteLLM API key in the authentication field so it can connect to the proxy.
![Enter API key](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/f6cfcb81-021d-4a41-94d7-d4eaf449d025/ascreenshot_d635865abfb64732a7278922f08dbcaa_text_export.jpeg)
Click **"Connect"** to establish the connection.
![Click Connect](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/1146b326-6f0c-4050-9729-af5c88e1bc81/ascreenshot_e19fb857e5394b9a9bf77b075b4fb620_text_export.jpeg)
ChatGPT connects and shows the available tools. Since both DeepWiki and Exa are currently marked as public, ChatGPT can see tools from both servers.
![ChatGPT shows available MCP tools](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/43ac56b7-9933-4762-903a-370fc52c79b5/ascreenshot_39073d6dc3bc4bb6a79d93365a26a4f8_text_export.jpeg)
---
### Flow 2: Make an Existing Server Private (Exa)
Now let's do the reverse — take an existing MCP server (Exa) that's currently public and restrict it to internal access only. After this change, ChatGPT should no longer see Exa's tools.
#### Step 1: Edit the Server
Go to the MCP Servers table and click on the Exa server to open its detail view.
![Exa server overview](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/65844f13-b1ec-4092-b3fd-b1cae3c0c833/ascreenshot_cc8ea435c5e14761a1394ca80fe817c0_text_export.jpeg)
Switch to the **"Settings"** tab to access the edit form.
![Click Settings](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/d5b65271-561e-4d2a-b832-96d32611f6e4/ascreenshot_a200942b17264c1eb7a3ffdb2c2141f5_text_export.jpeg)
The edit form loads with Exa's current configuration.
![Edit server](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/119184f6-f3cd-45b7-9cfa-0ea08de27020/ascreenshot_c39a793da03a4f0fb84b5ee829af9034_text_export.jpeg)
#### Step 2: Toggle Off "Available on Public Internet"
Scroll down and expand the **Permission Management / Access Control** section to find the public internet toggle.
![Expand permissions](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/bf7114cc-8741-4fa0-a39a-fe625482e88a/ascreenshot_8a987649c03e46558a2ec9a6f2f539a4_text_export.jpeg)
Toggle **"Available on Public Internet"** off. This will hide Exa from any caller outside your private network.
![Toggle off public internet](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/f36af5ad-028f-4bb1-aed1-43e38ff9b733/ascreenshot_9128364a049f489bb8483e18e5c88015_text_export.jpeg)
Click **"Save Changes"** to apply. The change takes effect immediately — no proxy restart needed.
![Save changes](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/126a71b3-02e1-4d61-a208-942b92e9ef25/ascreenshot_f349ef69e08044dd8e4903f4286b7b97_text_export.jpeg)
#### Step 3: Verify in ChatGPT
Go back to ChatGPT to confirm Exa is no longer visible. You'll need to reconnect for ChatGPT to re-fetch the tool list.
![ChatGPT verify](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/15518882-8b19-44d3-9bba-245aeb62b4b1/ascreenshot_f98f59c51e6543e1be4f3960ba375fc9_text_export.jpeg)
Open the MCP server settings and select to add or reconnect a server.
![Reconnect to server](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/784d3174-77c0-42e6-a059-4c906db8f72a/ascreenshot_d77db951b83e4b15a00373222712f6b5_text_export.jpeg)
Enter the same LiteLLM MCP URL as before.
![Reconnect URL](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/17ef5fb0-b240-4556-8d20-753d359b7fcf/ascreenshot_583466ce9e8f40d1ba0af8b1e7d04413_text_export.jpeg)
Set the server label.
![Reconnect name](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/d7907637-c957-4a3c-ab4f-1600ca9a70a0/ascreenshot_e429eea43f3f4b3ca4d3ac5a77fbde2d_text_export.jpeg)
Enter your API key for authentication.
![Reconnect key](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/9cfff77a-37aa-4ca6-8032-0b46c50f37e3/ascreenshot_250664183399496b8f5c9f86f576fc0b_text_export.jpeg)
Click **"Connect"** to re-establish the connection.
![Click Connect](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/686f6307-b4ae-448b-ac6c-2c9d7b4f6b57/ascreenshot_3f499d0812af42ab89fed103cc21c249_text_export.jpeg)
This time, only DeepWiki's tools appear — Exa is gone. LiteLLM detected that ChatGPT is calling from a public IP and filtered out Exa since it's no longer marked as public. Internal users on your private network would still see both servers.
![Only DeepWiki tools visible](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/667d79b6-75f9-4799-9315-0c176e7a5e34/ascreenshot_efa43050ac0b4445a09e542fa8f270ff_text_export.jpeg)
## Configuration Reference
### Per-Server Setting
<Tabs>
<TabItem value="ui" label="UI">
Toggle **"Available on Public Internet"** in the Permission Management section when creating or editing an MCP server.
</TabItem>
<TabItem value="config" label="config.yaml">
```yaml title="config.yaml" showLineNumbers
mcp_servers:
deepwiki:
url: https://mcp.deepwiki.com/mcp
available_on_public_internet: true # visible to external callers
exa:
url: https://exa.ai/mcp
auth_type: api_key
auth_value: os.environ/EXA_API_KEY
available_on_public_internet: false # internal only (default)
```
</TabItem>
<TabItem value="api" label="API">
```bash title="Create a public MCP server" showLineNumbers
curl -X POST <your-litellm-url>/v1/mcp/server \
-H "Authorization: Bearer sk-..." \
-H "Content-Type: application/json" \
-d '{
"server_name": "DeepWiki",
"url": "https://mcp.deepwiki.com/mcp",
"transport": "http",
"available_on_public_internet": true
}'
```
```bash title="Update an existing server" showLineNumbers
curl -X PUT <your-litellm-url>/v1/mcp/server \
-H "Authorization: Bearer sk-..." \
-H "Content-Type: application/json" \
-d '{
"server_id": "<server-id>",
"available_on_public_internet": false
}'
```
</TabItem>
</Tabs>
### Custom Private IP Ranges
By default, LiteLLM treats RFC 1918 private ranges as internal. You can customize this in the **Network Settings** tab under MCP Servers, or via config:
```yaml title="config.yaml" showLineNumbers
general_settings:
mcp_internal_ip_ranges:
- "10.0.0.0/8"
- "172.16.0.0/12"
- "192.168.0.0/16"
- "100.64.0.0/10" # Add your VPN/Tailscale range
```
When empty, the standard private ranges are used (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `127.0.0.0/8`).

27
docs/my-website/docs/providers/elevenlabs.md
View file

@ -243,6 +243,13 @@ ElevenLabs provides high-quality text-to-speech capabilities through their TTS A
| Supported Operations | `/audio/speech` |
| Link to Provider Doc | [ElevenLabs TTS API ↗](https://elevenlabs.io/docs/api-reference/text-to-speech) |
### Supported Models
| Model | Route | Description |
|-------|-------|-------------|
| Eleven v3 | `elevenlabs/eleven_v3` | Most expressive model. 70+ languages, audio tags support for sound effects and pauses. |
| Eleven Multilingual v2 | `elevenlabs/eleven_multilingual_v2` | Default TTS model. 29 languages, stable and production-ready. |
### Quick Start
#### LiteLLM Python SDK
@ -265,6 +272,26 @@ with open("test_output.mp3", "wb") as f:
f.write(audio.read())
```
#### Using Eleven v3 with Audio Tags
Eleven v3 supports [audio tags](https://elevenlabs.io/docs/overview/capabilities/text-to-speech#audio-tags) for adding sound effects and pauses directly in the text:
```python showLineNumbers title="Eleven v3 with audio tags"
import litellm
import os
os.environ["ELEVENLABS_API_KEY"] = "your-elevenlabs-api-key"
audio = litellm.speech(
model="elevenlabs/eleven_v3",
input='Welcome back. <sfx>applause</sfx> Today we have a special guest. <pause duration="1.5s"/> Let me introduce them.',
voice="alloy",
)
with open("eleven_v3_output.mp3", "wb") as f:
f.write(audio.read())
```
#### Advanced Usage: Overriding Parameters and ElevenLabs-Specific Features
```python showLineNumbers title="Advanced TTS with custom parameters"

391
docs/my-website/docs/proxy/cli.md
View file

@ -1,7 +1,10 @@
# CLI Arguments
Cli arguments, --host, --port, --num_workers
## --host
This page documents all command-line interface (CLI) arguments available for the LiteLLM proxy server.
## Server Configuration
### --host
- **Default:** `'0.0.0.0'`
- The host for the server to listen on.
- **Usage:**
@ -14,7 +17,7 @@ Cli arguments, --host, --port, --num_workers
litellm
```
## --port
### --port
- **Default:** `4000`
- The port to bind the server to.
- **Usage:**
@ -27,9 +30,9 @@ Cli arguments, --host, --port, --num_workers
litellm
```
## --num_workers
- **Default:** `1`
- The number of uvicorn workers to spin up.
### --num_workers
- **Default:** Number of logical CPUs in the system, or `4` if that cannot be determined
- The number of uvicorn / gunicorn workers to spin up.
- **Usage:**
```shell
litellm --num_workers 4
@ -40,55 +43,273 @@ Cli arguments, --host, --port, --num_workers
litellm
```
## --api_base
### --config
- **Short form:** `-c`
- **Default:** `None`
- The API base for the model litellm should call.
- Path to the proxy configuration file (e.g., config.yaml).
- **Usage:**
```shell
litellm --config path/to/config.yaml
```
### --log_config
- **Default:** `None`
- **Type:** `str`
- Path to the logging configuration file for uvicorn.
- **Usage:**
```shell
litellm --log_config path/to/log_config.conf
```
### --keepalive_timeout
- **Default:** `None`
- **Type:** `int`
- Set the uvicorn keepalive timeout in seconds (uvicorn timeout_keep_alive parameter).
- **Usage:**
```shell
litellm --keepalive_timeout 30
```
- **Usage - set Environment Variable:** `KEEPALIVE_TIMEOUT`
```shell
export KEEPALIVE_TIMEOUT=30
litellm
```
### --max_requests_before_restart
- **Default:** `None`
- **Type:** `int`
- Restart worker after this many requests. This is useful for mitigating memory growth over time.
- For uvicorn: maps to `limit_max_requests`
- For gunicorn: maps to `max_requests`
- **Usage:**
```shell
litellm --max_requests_before_restart 10000
```
- **Usage - set Environment Variable:** `MAX_REQUESTS_BEFORE_RESTART`
```shell
export MAX_REQUESTS_BEFORE_RESTART=10000
litellm
```
## Server Backend Options
### --run_gunicorn
- **Default:** `False`
- **Type:** `bool` (Flag)
- Starts proxy via gunicorn instead of uvicorn. Better for managing multiple workers in production.
- **Usage:**
```shell
litellm --run_gunicorn
```
### --run_hypercorn
- **Default:** `False`
- **Type:** `bool` (Flag)
- Starts proxy via hypercorn instead of uvicorn. Supports HTTP/2.
- **Usage:**
```shell
litellm --run_hypercorn
```
### --skip_server_startup
- **Default:** `False`
- **Type:** `bool` (Flag)
- Skip starting the server after setup (useful for database migrations only).
- **Usage:**
```shell
litellm --skip_server_startup
```
## SSL/TLS Configuration
### --ssl_keyfile_path
- **Default:** `None`
- **Type:** `str`
- Path to the SSL keyfile. Use this when you want to provide SSL certificate when starting proxy.
- **Usage:**
```shell
litellm --ssl_keyfile_path /path/to/key.pem --ssl_certfile_path /path/to/cert.pem
```
- **Usage - set Environment Variable:** `SSL_KEYFILE_PATH`
```shell
export SSL_KEYFILE_PATH=/path/to/key.pem
litellm
```
### --ssl_certfile_path
- **Default:** `None`
- **Type:** `str`
- Path to the SSL certfile. Use this when you want to provide SSL certificate when starting proxy.
- **Usage:**
```shell
litellm --ssl_certfile_path /path/to/cert.pem --ssl_keyfile_path /path/to/key.pem
```
- **Usage - set Environment Variable:** `SSL_CERTFILE_PATH`
```shell
export SSL_CERTFILE_PATH=/path/to/cert.pem
litellm
```
### --ciphers
- **Default:** `None`
- **Type:** `str`
- Ciphers to use for the SSL setup. Only used with `--run_hypercorn`.
- **Usage:**
```shell
litellm --run_hypercorn --ssl_keyfile_path /path/to/key.pem --ssl_certfile_path /path/to/cert.pem --ciphers "ECDHE+AESGCM"
```
## Model Configuration
### --model or -m
- **Default:** `None`
- The model name to pass to LiteLLM.
- **Usage:**
```shell
litellm --model gpt-3.5-turbo
```
### --alias
- **Default:** `None`
- An alias for the model, for user-friendly reference. Use this to give a litellm model name (e.g., "huggingface/codellama/CodeLlama-7b-Instruct-hf") a more user-friendly name ("codellama").
- **Usage:**
```shell
litellm --alias my-gpt-model
```
### --api_base
- **Default:** `None`
- The API base for the model LiteLLM should call.
- **Usage:**
```shell
litellm --model huggingface/tinyllama --api_base https://k58ory32yinf1ly0.us-east-1.aws.endpoints.huggingface.cloud
```
## --api_version
- **Default:** `None`
### --api_version
- **Default:** `2024-07-01-preview`
- For Azure services, specify the API version.
- **Usage:**
```shell
litellm --model azure/gpt-deployment --api_version 2023-08-01 --api_base https://<your api base>"
```
## --model or -m
### --headers
- **Default:** `None`
- The model name to pass to Litellm.
- Headers for the API call (as JSON string).
- **Usage:**
```shell
litellm --model gpt-3.5-turbo
litellm --model my-model --headers '{"Authorization": "Bearer token"}'
```
## --test
- **Type:** `bool` (Flag)
- Proxy chat completions URL to make a test request.
- **Usage:**
```shell
litellm --test
```
## --health
- **Type:** `bool` (Flag)
- Runs a health check on all models in config.yaml
- **Usage:**
```shell
litellm --health
```
## --alias
### --add_key
- **Default:** `None`
- An alias for the model, for user-friendly reference.
- Add a key to the model configuration.
- **Usage:**
```shell
litellm --alias my-gpt-model
litellm --add_key my-api-key
```
## --debug
### --save
- **Type:** `bool` (Flag)
- Save the model-specific config.
- **Usage:**
```shell
litellm --model gpt-3.5-turbo --save
```
## Model Parameters
### --temperature
- **Default:** `None`
- **Type:** `float`
- Set the temperature for the model.
- **Usage:**
```shell
litellm --temperature 0.7
```
### --max_tokens
- **Default:** `None`
- **Type:** `int`
- Set the maximum number of tokens for the model output.
- **Usage:**
```shell
litellm --max_tokens 50
```
### --request_timeout
- **Default:** `None`
- **Type:** `int`
- Set the timeout in seconds for completion calls.
- **Usage:**
```shell
litellm --request_timeout 300
```
### --max_budget
- **Default:** `None`
- **Type:** `float`
- Set max budget for API calls. Works for hosted models like OpenAI, TogetherAI, Anthropic, etc.
- **Usage:**
```shell
litellm --max_budget 100.0
```
### --drop_params
- **Type:** `bool` (Flag)
- Drop any unmapped params.
- **Usage:**
```shell
litellm --drop_params
```
### --add_function_to_prompt
- **Type:** `bool` (Flag)
- If a function passed but unsupported, pass it as a part of the prompt.
- **Usage:**
```shell
litellm --add_function_to_prompt
```
## Database Configuration
### --iam_token_db_auth
- **Default:** `False`
- **Type:** `bool` (Flag)
- Connects to an RDS database using IAM token authentication instead of a password. This is useful for AWS RDS instances that are configured to use IAM database authentication.
- When enabled, LiteLLM will generate an IAM authentication token to connect to the database.
- **Required Environment Variables:**
- `DATABASE_HOST` - The RDS database host
- `DATABASE_PORT` - The database port
- `DATABASE_USER` - The database user
- `DATABASE_NAME` - The database name
- `DATABASE_SCHEMA` (optional) - The database schema
- **Usage:**
```shell
litellm --iam_token_db_auth
```
- **Usage - set Environment Variable:** `IAM_TOKEN_DB_AUTH`
```shell
export IAM_TOKEN_DB_AUTH=True
export DATABASE_HOST=mydb.us-east-1.rds.amazonaws.com
export DATABASE_PORT=5432
export DATABASE_USER=mydbuser
export DATABASE_NAME=mydb
litellm
```
### --use_prisma_db_push
- **Default:** `False`
- **Type:** `bool` (Flag)
- Use `prisma db push` instead of `prisma migrate` for database schema updates. This is useful when you want to quickly sync your database schema without creating migration files.
- **Usage:**
```shell
litellm --use_prisma_db_push
```
## Debugging
### --debug
- **Default:** `False`
- **Type:** `bool` (Flag)
- Enable debugging mode for the input.
@ -102,10 +323,10 @@ Cli arguments, --host, --port, --num_workers
litellm
```
## --detailed_debug
### --detailed_debug
- **Default:** `False`
- **Type:** `bool` (Flag)
- Enable debugging mode for the input.
- Enable detailed debugging mode to view verbose debug logs.
- **Usage:**
```shell
litellm --detailed_debug
@ -116,80 +337,76 @@ Cli arguments, --host, --port, --num_workers
litellm
```
#### --temperature
- **Default:** `None`
- **Type:** `float`
- Set the temperature for the model.
- **Usage:**
```shell
litellm --temperature 0.7
```
## --max_tokens
- **Default:** `None`
- **Type:** `int`
- Set the maximum number of tokens for the model output.
- **Usage:**
```shell
litellm --max_tokens 50
```
## --request_timeout
- **Default:** `6000`
- **Type:** `int`
- Set the timeout in seconds for completion calls.
- **Usage:**
```shell
litellm --request_timeout 300
```
## --drop_params
### --local
- **Default:** `False`
- **Type:** `bool` (Flag)
- Drop any unmapped params.
- For local debugging purposes.
- **Usage:**
```shell
litellm --drop_params
litellm --local
```
## --add_function_to_prompt
## Testing & Health Checks
### --test
- **Type:** `bool` (Flag)
- If a function passed but unsupported, pass it as a part of the prompt.
- Proxy chat completions URL to make a test request to.
- **Usage:**
```shell
litellm --add_function_to_prompt
litellm --test
```
## --config
- Configure Litellm by providing a configuration file path.
### --test_async
- **Default:** `False`
- **Type:** `bool` (Flag)
- Calls async endpoints `/queue/requests` and `/queue/response`.
- **Usage:**
```shell
litellm --config path/to/config.yaml
litellm --test_async
```
## --telemetry
### --num_requests
- **Default:** `10`
- **Type:** `int`
- Number of requests to hit async endpoint with (used with `--test_async`).
- **Usage:**
```shell
litellm --test_async --num_requests 100
```
### --health
- **Type:** `bool` (Flag)
- Runs a health check on all models in config.yaml.
- **Usage:**
```shell
litellm --health
```
## Other Options
### --version
- **Short form:** `-v`
- **Type:** `bool` (Flag)
- Print LiteLLM version and exit.
- **Usage:**
```shell
litellm --version
```
### --telemetry
- **Default:** `True`
- **Type:** `bool`
- Help track usage of this feature.
- Help track usage of this feature. Turn off for privacy.
- **Usage:**
```shell
litellm --telemetry False
```
## --log_config
- **Default:** `None`
- **Type:** `str`
- Specify a log configuration file for uvicorn.
- **Usage:**
```shell
litellm --log_config path/to/log_config.conf
```
## --skip_server_startup
### --use_queue
- **Default:** `False`
- **Type:** `bool` (Flag)
- Skip starting the server after setup (useful for DB migrations only).
- To use celery workers for async endpoints.
- **Usage:**
```shell
litellm --skip_server_startup
```
litellm --use_queue
```

58
docs/my-website/docs/proxy/guardrails/custom_code_guardrail.md
View file

@ -61,15 +61,23 @@ curl -X POST http://localhost:4000/chat/completions \
### Function Signature
Your code must define an `apply_guardrail` function:
Your code must define an `apply_guardrail` function. It can be either sync or async:
```python
# Sync version
def apply_guardrail(inputs, request_data, input_type):
# inputs: see table below
# request_data: {"model": "...", "user_id": "...", "team_id": "...", "metadata": {...}}
# input_type: "request" or "response"
return allow() # or block() or modify()
# Async version (recommended when using HTTP primitives)
async def apply_guardrail(inputs, request_data, input_type):
response = await http_post("https://api.example.com/check", body={"text": inputs["texts"][0]})
if response["success"] and response["body"].get("flagged"):
return block("Content flagged")
return allow()
```
### `inputs` Parameter
@ -145,6 +153,29 @@ def apply_guardrail(inputs, request_data, input_type):
| `char_count(text)` | Count characters |
| `lower(text)` / `upper(text)` / `trim(text)` | String transforms |
### HTTP Requests (Async)
Make async HTTP requests to external APIs for additional validation or content moderation.
| Function | Description |
|----------|-------------|
| `await http_request(url, method, headers, body, timeout)` | General async HTTP request |
| `await http_get(url, headers, timeout)` | Async GET request |
| `await http_post(url, body, headers, timeout)` | Async POST request |
**Response format:**
```python
{
"status_code": 200, # HTTP status code
"body": {...}, # Response body (parsed JSON or string)
"headers": {...}, # Response headers
"success": True, # True if status code is 2xx
"error": None # Error message if request failed
}
```
**Note:** When using HTTP primitives, define your function as `async def apply_guardrail(...)` for non-blocking execution.
## Examples
### Block PII (SSN)
@ -213,6 +244,29 @@ def apply_guardrail(inputs, request_data, input_type):
return allow()
```
### Call External Moderation API (Async)
```python
async def apply_guardrail(inputs, request_data, input_type):
# Call an external moderation API
for text in inputs["texts"]:
response = await http_post(
"https://api.example.com/moderate",
body={"text": text, "user_id": request_data["user_id"]},
headers={"Authorization": "Bearer YOUR_API_KEY"},
timeout=10
)
if not response["success"]:
# API call failed - decide whether to allow or block
return allow()
if response["body"].get("flagged"):
return block(response["body"].get("reason", "Content flagged"))
return allow()
```
### Combine Multiple Checks
```python
@ -241,8 +295,8 @@ Custom code runs in a restricted environment:
- ❌ No `import` statements
- ❌ No file I/O
- ❌ No network access
- ❌ No `exec()` or `eval()`
- ✅ HTTP requests via built-in `http_request`, `http_get`, `http_post` primitives
- ✅ Only LiteLLM-provided primitives available
## Per-Request Usage

130
docs/my-website/docs/proxy/ui_team_soft_budget_alerts.md Normal file
View file

@ -0,0 +1,130 @@
import Image from '@theme/IdealImage';
# Team Soft Budget Alerts
Set a soft budget on a team and get email alerts when spending crosses the threshold — without blocking any requests.
## Overview
A **soft budget** is a spending threshold that triggers email notifications when exceeded, but **does not block requests**. This is different from a hard budget (`max_budget`), which rejects requests once the limit is reached.
<Image img={require('../../img/ui_team_soft_budget_alerts.png')} />
Team soft budget alerts let you:
- **Get notified early** — receive email alerts when a team's spend crosses the soft budget threshold
- **Keep requests flowing** — unlike hard budgets, soft budgets never block API calls
- **Target specific recipients** — send alerts to specific email addresses (e.g. team leads, finance), not just the team members
- **Work without global alerting** — team soft budget alerts are sent via email independently of Slack or other global alerting configuration
:::warning Email integration required
Team soft budget alerts are sent via email. You must have an active email integration (SendGrid, Resend, or SMTP) configured on your proxy for alerts to be delivered. See [Email Notifications](./email.md) for setup instructions.
:::
:::info Automatically active
Team soft budget alerts are **automatically active** once you configure a soft budget and at least one alerting email on a team. No additional proxy configuration or restart is needed — alerts are checked on every request.
:::
## How It Works
On every API request made with a key belonging to a team, the proxy checks:
1. Does the team have a `soft_budget` set?
2. Is the team's current `spend` >= the `soft_budget`?
3. Are there any emails configured in `soft_budget_alerting_emails`?
If all three conditions are met, an email alert is sent to the configured recipients. Alerts are **deduplicated** so the same alert is only sent once within a 24-hour window.
## How to Set Up Team Soft Budget Alerts
### 1. Navigate to the Admin UI
Go to the Admin UI (e.g. `http://localhost:4000/ui` or your `PROXY_BASE_URL/ui`).
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/f06d75ad-25ef-4ee8-90c3-9604f8e46a1c/ascreenshot_1a6defaed1494d6da0001459511ecfd5_text_export.jpeg)
### 2. Go to Teams
Click **Teams** in the sidebar.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/f06d75ad-25ef-4ee8-90c3-9604f8e46a1c/ascreenshot_2d258fa280f6463b966bf7a05bb102d5_text_export.jpeg)
### 3. Select a team
Click on the team you want to configure soft budget alerts for.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/490f09fb-6bf5-45a8-a384-676889f34c88/ascreenshot_15cceb22abe64df0bf7d7c742ecb5b2f_text_export.jpeg)
### 4. Open team Settings
Click the **Settings** tab to view the team's configuration.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/28dd1bc5-7d07-462f-b277-33f885bdc07e/ascreenshot_12f2b762b5d24686801d93ad5b067e06_text_export.jpeg)
### 5. Edit Settings
Click **Edit Settings** to modify the team's budget configuration.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/30a483ea-7e01-4fdc-ac5f-a5572388d138/ascreenshot_0915eadd9e754a798489853b82de3cb5_text_export.jpeg)
### 6. Set the Soft Budget
Click the **Soft Budget (USD)** field and enter your desired threshold. For example, enter `0.01` for testing or a higher value like `500` for production.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/8b306d80-4943-4ad0-a51a-94b5ebdd6680/ascreenshot_5bb6e65c6428473fac2607f6a7f4b98a_text_export.jpeg)
### 7. Add alerting emails
Click the **Soft Budget Alerting Emails** field and enter one or more comma-separated email addresses that should receive the alert.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/a97c6efa-cc93-45d7-979e-d2a533f423b9/ascreenshot_2d8223ce8e934aa1bfadfb2f78aee5fc_text_export.jpeg)
### 8. Save Changes
Click **Save Changes**. The soft budget alert is now active — no proxy restart required.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/865ba6f1-3fc6-4c19-8e08-433561d6c3f7/ascreenshot_b2f0503ada3a479a83dc8b7d01c1f8da_text_export.jpeg)
### 9. Verify: email alert received
Once the team's spend crosses the soft budget, an email alert is sent to the configured recipients. Below is an example of the alert email:
<Image img={require('../../img/ui_team_soft_budget_email_example.png')} />
## Settings Reference
| Setting | Description |
| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
| **Soft Budget (USD)** | The spending threshold that triggers an email alert. Requests are **not** blocked when this limit is exceeded. |
| **Soft Budget Alerting Emails** | Comma-separated email addresses that receive the alert when the soft budget is crossed. At least one email is required for alerts to be sent. |
:::tip Soft Budget vs. Max Budget
- **Soft Budget**: Advisory threshold — sends email alerts but does **not** block requests.
- **Max Budget**: Hard limit — blocks requests once the budget is exceeded.
You can set both on the same team to get early warnings (soft) and a hard stop (max).
:::
## API Configuration
You can also configure team soft budgets via the API when creating or updating a team:
```bash
curl -X POST 'http://localhost:4000/team/update' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"team_id": "your-team-id",
"soft_budget": 500.00,
"metadata": {
"soft_budget_alerting_emails": ["lead@example.com", "finance@example.com"]
}
}'
```
## Related Documentation
- [Email Notifications](./email.md) – Configure email integrations (Resend, SMTP) for LiteLLM Proxy
- [Alerting](./alerting.md) – Set up Slack and other alerting channels
- [Cost Tracking](./cost_tracking.md) – Track and manage spend across teams, keys, and users

52
docs/my-website/docs/troubleshoot.md
View file

@ -45,6 +45,58 @@ Full error logs, stack traces, and any images from service metrics (CPU, memory,
---
## UI Issues
If you're experiencing issues with the LiteLLM Admin UI, please include the following information in addition to the general details above.
### 1. Steps to Reproduce
A clear, step-by-step description of how to trigger the issue (e.g., "Navigate to Settings → Team, click 'Create Team', fill in fields, click submit → error appears").
### 2. LiteLLM Version
The current version of LiteLLM you're running. Check via `litellm --version` or the UI's settings page.
### 3. Architecture & Deployment Setup
Distributed environments are a known source of UI issues. Please describe:
- **Number of LiteLLM instances/replicas** and how they are deployed (e.g., Kubernetes, Docker Compose, ECS)
- **Load balancer** type and configuration (e.g., ALB, Nginx, Cloudflare Tunnel) — include whether sticky sessions are enabled
- **How the UI is accessed** — directly via LiteLLM, through a reverse proxy, or behind an ingress controller
- **Any CDN or caching layers** between the user and the LiteLLM server
### 4. Network Tab Requests
Open your browser's Developer Tools (F12 → Network tab), reproduce the issue, and share:
- The **failing request(s)** — URL, method, status code, and response body
- **Screenshots or HAR export** of the relevant network activity
- Any **CORS or mixed-content errors** shown in the Console tab
### 5. Environment Variables
Non-sensitive environment variables related to the UI and proxy setup, such as:
- `LITELLM_MASTER_KEY`
- `PROXY_BASE_URL` / `LITELLM_PROXY_BASE_URL`
- `UI_BASE_PATH`
- Any SSO-related variables (e.g., `GOOGLE_CLIENT_ID`, `MICROSOFT_TENANT`)
Do **not** include passwords, secrets, or API keys.
### 6. Browser & Access Details
- **Browser** and version (e.g., Chrome 120, Firefox 121)
- **Access URL** used to reach the UI (redact sensitive parts)
- Whether the issue occurs for **all users or specific roles** (Admin, Internal User, etc.)
### 7. Screenshots or Screen Recordings
A screenshot or short screen recording of the issue is extremely helpful. Include any visible error messages, toasts, or unexpected behavior.
---
## Support Channels
[Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)

129
docs/my-website/docs/tutorials/claude_code_beta_headers.md Normal file
View file

@ -0,0 +1,129 @@
import Image from '@theme/IdealImage';
# Claude Code - Fixing Invalid Beta Header Errors
When using Claude Code with LiteLLM and non-Anthropic providers (Bedrock, Azure AI, Vertex AI), you may encounter "invalid beta header" errors. This guide explains how to fix these errors locally or contribute a fix to LiteLLM.
## What Are Beta Headers?
Anthropic uses beta headers to enable experimental features in Claude. When you use Claude Code, it may send beta headers like:
```
anthropic-beta: prompt-caching-scope-2026-01-05,advanced-tool-use-2025-11-20
```
However, not all providers support all Anthropic beta features. When an unsupported beta header is sent to a provider, you'll see an error.
## Common Error Message
```bash
Error: The model returned the following errors: invalid beta flag
```
## How LiteLLM Handles Beta Headers
LiteLLM automatically filters out unsupported beta headers using a configuration file:
```
litellm/litellm/anthropic_beta_headers_config.json
```
This JSON file lists which beta headers are **unsupported** for each provider. Headers not in the unsupported list are passed through to the provider.
## Quick Fix: Update Config Locally
If you encounter an invalid beta header error, you can fix it immediately by updating the config file locally.
### Step 1: Locate the Config File
Find the file in your LiteLLM installation:
```bash
# If installed via pip
cd $(python -c "import litellm; import os; print(os.path.dirname(litellm.__file__))")
# The config file is at:
# litellm/anthropic_beta_headers_config.json
```
### Step 2: Add the Unsupported Header
Open `anthropic_beta_headers_config.json` and add the problematic header to the appropriate provider's list:
```json title="anthropic_beta_headers_config.json"
{
"description": "Unsupported Anthropic beta headers for each provider. Headers listed here will be dropped. Headers not listed are passed through as-is.",
"anthropic": [],
"azure_ai": [],
"bedrock_converse": [
"prompt-caching-scope-2026-01-05",
"bash_20250124",
"bash_20241022",
"text_editor_20250124",
"text_editor_20241022",
"compact-2026-01-12",
"advanced-tool-use-2025-11-20",
"web-fetch-2025-09-10",
"code-execution-2025-08-25",
"skills-2025-10-02",
"files-api-2025-04-14"
],
"bedrock": [
"advanced-tool-use-2025-11-20",
"prompt-caching-scope-2026-01-05",
"structured-outputs-2025-11-13",
"web-fetch-2025-09-10",
"code-execution-2025-08-25",
"skills-2025-10-02",
"files-api-2025-04-14"
],
"vertex_ai": [
"prompt-caching-scope-2026-01-05"
]
}
```
### Step 3: Restart Your Application
After updating the config file, restart your LiteLLM proxy or application:
```bash
# If using LiteLLM proxy
litellm --config config.yaml
# If using Python SDK
# Just restart your Python application
```
The updated configuration will be loaded automatically.
## Contributing a Fix to LiteLLM
Help the community by contributing your fix! If your local changes work, please raise a PR with the addition of the header and we will merge it.
## How Beta Header Filtering Works
When you make a request through LiteLLM:
```mermaid
sequenceDiagram
participant CC as Claude Code
participant LP as LiteLLM
participant Config as Beta Headers Config
participant Provider as Provider (Bedrock/Azure/etc)
CC->>LP: Request with beta headers
Note over CC,LP: anthropic-beta: header1,header2,header3
LP->>Config: Load unsupported headers for provider
Config-->>LP: Returns unsupported list
Note over LP: Filter headers:<br/>- Remove unsupported<br/>- Keep supported
LP->>Provider: Request with filtered headers
Note over LP,Provider: anthropic-beta: header2<br/>(header1, header3 removed)
Provider-->>LP: Success response
LP-->>CC: Response
```

BIN
docs/my-website/img/release_notes/mcp_internet.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 724 KiB

BIN
docs/my-website/img/ui_team_soft_budget_alerts.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 328 KiB

BIN
docs/my-website/img/ui_team_soft_budget_email_example.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 50 KiB

2
docs/my-website/release_notes/v1.81.6.md
View file

@ -1,5 +1,5 @@
---
title: "v1.81.6 - Logs v2 with Tool Call Tracing"
title: "[Preview] v1.81.6 - Logs v2 with Tool Call Tracing"
slug: "v1-81-6"
date: 2026-01-31T00:00:00
authors:

372
docs/my-website/release_notes/v1.81.9.md Normal file
View file

@ -0,0 +1,372 @@
---
title: "[Preview] v1.81.9 - Control which MCP Servers are exposed on the Internet"
slug: "v1-81-9"
date: 2026-02-07T00:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
## Deploy this version
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.81.9.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.81.9
```
</TabItem>
</Tabs>
## Key Highlights
- **Claude Opus 4.6** - [Full support across Anthropic, AWS Bedrock, Azure AI, and Vertex AI with adaptive thinking and 1M context window](../../blog/claude_opus_4_6)
- **A2A Agent Gateway** - [Call A2A (Agent-to-Agent) registered agents through the standard `/chat/completions` API](../../docs/a2a_invoking_agents)
- **Expose MCP servers on the public internet** - [Launch MCP servers with public/private visibility and IP-based access control for internet-facing deployments](../../docs/mcp_public_internet)
- **UI Team Soft Budget Alerts** - [Set soft budgets on teams and receive email alerts when spending crosses the threshold — without blocking requests](../../docs/proxy/ui_team_soft_budget_alerts)
- **Performance Optimizations** - Multiple performance improvements including ~40% Prometheus CPU reduction, LRU caching, and optimized logging paths
- **LiteLLM Observatory** - [Automated 24-hour load tests](../../blog/litellm-observatory)
- **30% Faster Request Processing for Callback-Heavy Deployments** - [Performance improvement for callback heavy deployments][PR #20354](https://github.com/BerriAI/litellm/pull/20354)
---
## 30% Faster Request Processing for Callback-Heavy Deployments
If you use logging callbacks like Langfuse, Datadog, or Prometheus, every request was paying an unnecessary cost: three loops that re-sorted your callbacks on every single request, even though the callback list hadn't changed. The more callbacks you had configured, the more time was wasted. We moved this work to happen once at startup instead of on every request. For deployments with the default callback set, this is a ~30% speedup in request setup. For deployments with many callbacks configured, the improvement is even larger.
---
## LiteLLM Observatory
LiteLLM Observatory is a long-running release-validation system we built to catch regressions before they reach users. The system is built to be extensible—you can add new tests, configure models and failure thresholds, and queue runs against any deployment. Our goal is to achieve 100% coverage of LiteLLM functionality through these tests. We run 24-hour load tests against our production deployments before all releases, surfacing issues like resource lifecycle bugs, OOMs, and CPU regressions that only appear under sustained load.
---
## MCP Servers on the Public Internet
This release makes it safe to expose MCP servers on the public internet by adding public/private visibility and IP-based access control. You can now run internet-facing MCP services while restricting access to trusted networks and keeping internal tools private.
[Get started](../../docs/mcp_public_internet)
<Image
img={require('../img/release_notes/mcp_internet.png')}
style={{ maxWidth: '900px', width: '100%' }}
/>
## UI Team Soft Budget Alerts
Set a soft budget on any team to receive email alerts when spending crosses the threshold — without blocking any requests. Configure the threshold and alerting emails directly from the Admin UI, with no proxy restart needed.
[Get started](../../docs/proxy/ui_team_soft_budget_alerts)
<Image
img={require('../img/ui_team_soft_budget_alerts.png')}
style={{ maxWidth: '900px', width: '100%' }}
/>
Let's dive in.
---
## New Models / Updated Models
#### New Model Support (13 new models)
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) |
| -------- | ----- | -------------- | ------------------- | -------------------- |
| Anthropic | `claude-opus-4-6` | 1M | $5.00 | $25.00 |
| AWS Bedrock | `anthropic.claude-opus-4-6-v1` | 1M | $5.00 | $25.00 |
| Azure AI | `azure_ai/claude-opus-4-6` | 200K | $5.00 | $25.00 |
| Vertex AI | `vertex_ai/claude-opus-4-6` | 1M | $5.00 | $25.00 |
| Google Gemini | `gemini/deep-research-pro-preview-12-2025` | 65K | $2.00 | $12.00 |
| Vertex AI | `vertex_ai/deep-research-pro-preview-12-2025` | 65K | $2.00 | $12.00 |
| Moonshot | `moonshot/kimi-k2.5` | 262K | $0.60 | $3.00 |
| OpenRouter | `openrouter/qwen/qwen3-235b-a22b-2507` | 262K | $0.07 | $0.10 |
| OpenRouter | `openrouter/qwen/qwen3-235b-a22b-thinking-2507` | 262K | $0.11 | $0.60 |
| Together AI | `together_ai/zai-org/GLM-4.7` | 200K | $0.45 | $2.00 |
| Together AI | `together_ai/moonshotai/Kimi-K2.5` | 256K | $0.50 | $2.80 |
| ElevenLabs | `elevenlabs/eleven_v3` | - | $0.18/1K chars | - |
| ElevenLabs | `elevenlabs/eleven_multilingual_v2` | - | $0.18/1K chars | - |
#### Features
- **[Anthropic](../../docs/providers/anthropic)**
- Full Claude Opus 4.6 support with adaptive thinking across all regions (us, eu, apac, au) - [PR #20506](https://github.com/BerriAI/litellm/pull/20506), [PR #20508](https://github.com/BerriAI/litellm/pull/20508), [PR #20514](https://github.com/BerriAI/litellm/pull/20514), [PR #20551](https://github.com/BerriAI/litellm/pull/20551)
- Map reasoning content to anthropic thinking block (streaming + non-streaming) - [PR #20254](https://github.com/BerriAI/litellm/pull/20254)
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Add 1hr tiered caching costs for long-context models - [PR #20214](https://github.com/BerriAI/litellm/pull/20214)
- Support TTL (1h) field in prompt caching for Bedrock Claude 4.5 models - [PR #20338](https://github.com/BerriAI/litellm/pull/20338)
- Add Nova Sonic speech-to-speech model support - [PR #20244](https://github.com/BerriAI/litellm/pull/20244)
- Fix empty assistant message for Converse API - [PR #20390](https://github.com/BerriAI/litellm/pull/20390)
- Fix content blocked handling - [PR #20606](https://github.com/BerriAI/litellm/pull/20606)
- **[Google Gemini / Vertex AI](../../docs/providers/gemini)**
- Add Gemini Deep Research model support - [PR #20406](https://github.com/BerriAI/litellm/pull/20406)
- Fix Vertex AI Gemini streaming content_filter handling - [PR #20105](https://github.com/BerriAI/litellm/pull/20105)
- Allow using OpenAI-style tools for `web_search` with Vertex AI/Gemini models - [PR #20280](https://github.com/BerriAI/litellm/pull/20280)
- Fix `supports_native_streaming` for Gemini and Vertex AI models - [PR #20408](https://github.com/BerriAI/litellm/pull/20408)
- Add mapping for responses tools in file IDs - [PR #20402](https://github.com/BerriAI/litellm/pull/20402)
- **[Cohere](../../docs/providers/cohere)**
- Support `dimensions` param for Cohere embed v4 - [PR #20235](https://github.com/BerriAI/litellm/pull/20235)
- **[Cerebras](../../docs/providers/cerebras)**
- Add reasoning param support for GPT OSS Cerebras - [PR #20258](https://github.com/BerriAI/litellm/pull/20258)
- **[Moonshot](../../docs/providers/moonshot)**
- Add Kimi K2.5 model entries - [PR #20273](https://github.com/BerriAI/litellm/pull/20273)
- **[OpenRouter](../../docs/providers/openrouter)**
- Add Qwen3-235B models - [PR #20455](https://github.com/BerriAI/litellm/pull/20455)
- **[Together AI](../../docs/providers/togetherai)**
- Add GLM-4.7 and Kimi-K2.5 models - [PR #20319](https://github.com/BerriAI/litellm/pull/20319)
- **[ElevenLabs](../../docs/providers/elevenlabs)**
- Add `eleven_v3` and `eleven_multilingual_v2` TTS models - [PR #20522](https://github.com/BerriAI/litellm/pull/20522)
- **[Vercel AI Gateway](../../docs/providers/vercel_ai_gateway)**
- Add missing capability flags to models - [PR #20276](https://github.com/BerriAI/litellm/pull/20276)
- **[GitHub Copilot](../../docs/providers/github_copilot)**
- Fix system prompts being dropped and auto-add required Copilot headers - [PR #20113](https://github.com/BerriAI/litellm/pull/20113)
- **[GigaChat](../../docs/providers/gigachat)**
- Fix incorrect merging of consecutive user messages for GigaChat provider - [PR #20341](https://github.com/BerriAI/litellm/pull/20341)
- **[xAI](../../docs/providers/xai_realtime)**
- Add xAI `/realtime` API support - works with LiveKit SDK - [PR #20381](https://github.com/BerriAI/litellm/pull/20381)
- **[OpenAI](../../docs/providers/openai)**
- Add `gpt-5-search-api` model and docs clarifications - [PR #20512](https://github.com/BerriAI/litellm/pull/20512)
### Bug Fixes
- **[Anthropic](../../docs/providers/anthropic)**
- Fix extra inputs not permitted error for `provider_specific_fields` - [PR #20334](https://github.com/BerriAI/litellm/pull/20334)
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Fix: Managed Batches inconsistent state management for list and cancel batches - [PR #20331](https://github.com/BerriAI/litellm/pull/20331)
- **[OpenAI Embeddings](../../docs/providers/openai)**
- Fix `open_ai_embedding_models` to have `custom_llm_provider` None - [PR #20253](https://github.com/BerriAI/litellm/pull/20253)
---
## LLM API Endpoints
#### Features
- **[Messages API](../../docs/providers/anthropic)**
- Filter unsupported Claude Code beta headers for non-Anthropic providers - [PR #20578](https://github.com/BerriAI/litellm/pull/20578)
- Fix inconsistent response format in `anthropic.messages.acreate()` when using non-Anthropic providers - [PR #20442](https://github.com/BerriAI/litellm/pull/20442)
- Fix 404 on `/api/event_logging/batch` endpoint that caused Claude Code "route not found" errors - [PR #20504](https://github.com/BerriAI/litellm/pull/20504)
- **[A2A Agent Gateway](../../docs/a2a)**
- Allow calling A2A agents through LiteLLM `/chat/completions` API - [PR #20358](https://github.com/BerriAI/litellm/pull/20358)
- Use A2A registered agents with `/chat/completions` - [PR #20362](https://github.com/BerriAI/litellm/pull/20362)
- Fix A2A agents deployed with localhost/internal URLs in their agent cards - [PR #20604](https://github.com/BerriAI/litellm/pull/20604)
- **[Files API](../../docs/providers/gemini)**
- Add support for delete and GET via file_id for Gemini - [PR #20329](https://github.com/BerriAI/litellm/pull/20329)
- **General**
- Add User-Agent customization support - [PR #19881](https://github.com/BerriAI/litellm/pull/19881)
- Fix search tools not found when using per-request routers - [PR #19818](https://github.com/BerriAI/litellm/pull/19818)
- Forward extra headers in chat - [PR #20386](https://github.com/BerriAI/litellm/pull/20386)
---
## Management Endpoints / UI
#### Features
- **SSO Configuration**
- SSO Config Team Mappings - [PR #20111](https://github.com/BerriAI/litellm/pull/20111)
- UI - SSO: Add Team Mappings - [PR #20299](https://github.com/BerriAI/litellm/pull/20299)
- Extract user roles from JWT access token for Keycloak compatibility - [PR #20591](https://github.com/BerriAI/litellm/pull/20591)
- **Auth / SDK**
- Add `proxy_auth` for auto OAuth2/JWT token management in SDK - [PR #20238](https://github.com/BerriAI/litellm/pull/20238)
- **Virtual Keys**
- Key `reset_spend` endpoint - [PR #20305](https://github.com/BerriAI/litellm/pull/20305)
- UI - Keys: Allowed Routes to Key Info and Edit Pages - [PR #20369](https://github.com/BerriAI/litellm/pull/20369)
- Add Key info endpoint object permission data - [PR #20407](https://github.com/BerriAI/litellm/pull/20407)
- Keys and Teams Router Setting + Allow Override of Router Settings - [PR #20205](https://github.com/BerriAI/litellm/pull/20205)
- **Teams & Budgets**
- Add `soft_budget` to Team Table + Create/Update Endpoints - [PR #20530](https://github.com/BerriAI/litellm/pull/20530)
- Team Soft Budget Email Alerts - [PR #20553](https://github.com/BerriAI/litellm/pull/20553)
- UI - Team Settings: Soft Budget + Alerting Emails - [PR #20634](https://github.com/BerriAI/litellm/pull/20634)
- UI - User Budget Page: Unlimited Budget Checkbox - [PR #20380](https://github.com/BerriAI/litellm/pull/20380)
- `/user/update` allow for `max_budget` resets - [PR #20375](https://github.com/BerriAI/litellm/pull/20375)
- **UI Improvements**
- Default Team Settings: Migrate to use Reusable Model Select - [PR #20310](https://github.com/BerriAI/litellm/pull/20310)
- Navbar: Option to Hide Community Engagement Buttons - [PR #20308](https://github.com/BerriAI/litellm/pull/20308)
- Show team alias on Models health page - [PR #20359](https://github.com/BerriAI/litellm/pull/20359)
- Admin Settings: Add option for Authentication for public AI Hub - [PR #20444](https://github.com/BerriAI/litellm/pull/20444)
- Adjust daily spend date filtering for user timezone - [PR #20472](https://github.com/BerriAI/litellm/pull/20472)
- **SCIM**
- Add base `/scim/v2` endpoint for SCIM resource discovery - [PR #20301](https://github.com/BerriAI/litellm/pull/20301)
- **Proxy CLI**
- CLI arguments for RDS IAM auth - [PR #20437](https://github.com/BerriAI/litellm/pull/20437)
#### Bugs
- Fix: Remove unnecessary key blocking on UI login that prevented access - [PR #20210](https://github.com/BerriAI/litellm/pull/20210)
- UI - Team Settings: Disable Global Guardrail Persistence - [PR #20307](https://github.com/BerriAI/litellm/pull/20307)
- UI - Model Info Page: Fix Input and Output Labels - [PR #20462](https://github.com/BerriAI/litellm/pull/20462)
- UI - Model Page: Column Resizing on Smaller Screens - [PR #20599](https://github.com/BerriAI/litellm/pull/20599)
- Fix `/key/list` `user_id` Empty String Edge Case - [PR #20623](https://github.com/BerriAI/litellm/pull/20623)
- Add array type checks for model, agent, and MCP hub data to prevent UI crashes - [PR #20469](https://github.com/BerriAI/litellm/pull/20469)
- Fix unique constraint on daily tables + logging when updates fail - [PR #20394](https://github.com/BerriAI/litellm/pull/20394)
---
## Logging / Guardrail / Prompt Management Integrations
#### Bug Fixes (3 fixes)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Fix Langfuse OTEL trace export failing when spans contain null attributes - [PR #20382](https://github.com/BerriAI/litellm/pull/20382)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Fix incorrect failure metrics labels causing miscounted error rates - [PR #20152](https://github.com/BerriAI/litellm/pull/20152)
- **[Slack Alerts](../../docs/proxy/alerting)**
- Fix Slack alert delivery failing for certain budget threshold configurations - [PR #20257](https://github.com/BerriAI/litellm/pull/20257)
#### Guardrails (7 updates)
- **Custom Code Guardrails**
- Add HTTP support to custom code guardrails + Unified guardrails for MCP + Agent guardrail support - [PR #20619](https://github.com/BerriAI/litellm/pull/20619)
- Custom Code Guardrails UI Playground - [PR #20377](https://github.com/BerriAI/litellm/pull/20377)
- **Team-Based Guardrails**
- Implement team-based isolation guardrails management - [PR #20318](https://github.com/BerriAI/litellm/pull/20318)
- **[OpenAI Moderations](../../docs/apply_guardrail)**
- Ensure OpenAI Moderations Guard works with OpenAI Embeddings - [PR #20523](https://github.com/BerriAI/litellm/pull/20523)
- **[GraySwan / Cygnal](../../docs/apply_guardrail)**
- Fix fail-open for GraySwan and pass metadata to Cygnal API endpoint - [PR #19837](https://github.com/BerriAI/litellm/pull/19837)
- **General**
- Check for `model_response_choices` before guardrail input - [PR #19784](https://github.com/BerriAI/litellm/pull/19784)
- Preserve streaming content on guardrail-sampled chunks - [PR #20027](https://github.com/BerriAI/litellm/pull/20027)
---
## Spend Tracking, Budgets and Rate Limiting
- **Support 0 cost models** - Allow zero-cost model entries for internal/free-tier models - [PR #20249](https://github.com/BerriAI/litellm/pull/20249)
---
## MCP Gateway (9 updates)
- **MCP Semantic Filtering** - Filter MCP tools using semantic similarity to reduce tool sprawl for LLM calls - [PR #20296](https://github.com/BerriAI/litellm/pull/20296), [PR #20316](https://github.com/BerriAI/litellm/pull/20316)
- **UI - MCP Semantic Filtering** - Add support for MCP Semantic Filtering configuration on UI - [PR #20454](https://github.com/BerriAI/litellm/pull/20454)
- **MCP IP-Based Access Control** - Set MCP servers as private/public available on internet with IP-based restrictions - [PR #20607](https://github.com/BerriAI/litellm/pull/20607), [PR #20620](https://github.com/BerriAI/litellm/pull/20620)
- **Fix MCP "Session not found" error** on VSCode reconnect - [PR #20298](https://github.com/BerriAI/litellm/pull/20298)
- **Fix OAuth2 'Capabilities: none' bug** for upstream MCP servers - [PR #20602](https://github.com/BerriAI/litellm/pull/20602)
- **Include Config Defined Search Tools** in `/search_tools/list` - [PR #20371](https://github.com/BerriAI/litellm/pull/20371)
- **UI - Search Tools**: Show Config Defined Search Tools - [PR #20436](https://github.com/BerriAI/litellm/pull/20436)
- **Ensure MCP permissions are enforced** when using JWT Auth - [PR #20383](https://github.com/BerriAI/litellm/pull/20383)
- **Fix `gcs_bucket_name` not being passed** correctly for MCP server storage configuration - [PR #20491](https://github.com/BerriAI/litellm/pull/20491)
---
## Performance / Loadbalancing / Reliability improvements (14 improvements)
- **Prometheus ~40% CPU reduction** - Parallelize budget metrics, fix caching bug, reduce CPU usage - [PR #20544](https://github.com/BerriAI/litellm/pull/20544)
- **Prevent closed client errors** by reverting httpx client caching - [PR #20025](https://github.com/BerriAI/litellm/pull/20025)
- **Avoid unnecessary Router creation** when no models or search tools are configured - [PR #20661](https://github.com/BerriAI/litellm/pull/20661)
- **Optimize `wrapper_async`** with `CallTypes` caching and reduced lookups - [PR #20204](https://github.com/BerriAI/litellm/pull/20204)
- **Cache `_get_relevant_args_to_use_for_logging()`** at module level - [PR #20077](https://github.com/BerriAI/litellm/pull/20077)
- **LRU cache for `normalize_request_route`** - [PR #19812](https://github.com/BerriAI/litellm/pull/19812)
- **Optimize `get_standard_logging_metadata`** with set intersection - [PR #19685](https://github.com/BerriAI/litellm/pull/19685)
- **Early-exit guards in `completion_cost`** for unused features - [PR #20020](https://github.com/BerriAI/litellm/pull/20020)
- **Optimize `get_litellm_params`** with sparse kwargs extraction - [PR #19884](https://github.com/BerriAI/litellm/pull/19884)
- **Guard debug log f-strings** and remove redundant dict copies - [PR #19961](https://github.com/BerriAI/litellm/pull/19961)
- **Replace enum construction with frozenset lookup** - [PR #20302](https://github.com/BerriAI/litellm/pull/20302)
- **Guard debug f-string in `update_environment_variables`** - [PR #20360](https://github.com/BerriAI/litellm/pull/20360)
- **Warn when budget lookup fails** to surface silent caching misses - [PR #20545](https://github.com/BerriAI/litellm/pull/20545)
- **Add INFO-level session reuse logging** per request for better observability - [PR #20597](https://github.com/BerriAI/litellm/pull/20597)
---
## Database Changes
### Schema Updates
| Table | Change Type | Description | PR | Migration |
| ----- | ----------- | ----------- | -- | --------- |
| `LiteLLM_TeamTable` | New Column | Added `allow_team_guardrail_config` boolean field for team-based guardrail isolation | [PR #20318](https://github.com/BerriAI/litellm/pull/20318) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260205091235_allow_team_guardrail_config/migration.sql) |
| `LiteLLM_DeletedTeamTable` | New Column | Added `allow_team_guardrail_config` boolean field | [PR #20318](https://github.com/BerriAI/litellm/pull/20318) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260205091235_allow_team_guardrail_config/migration.sql) |
| `LiteLLM_TeamTable` | New Column | Added `soft_budget` (double precision) for soft budget alerting | [PR #20530](https://github.com/BerriAI/litellm/pull/20530) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260205144610_add_soft_budget_to_team_table/migration.sql) |
| `LiteLLM_DeletedTeamTable` | New Column | Added `soft_budget` (double precision) | [PR #20653](https://github.com/BerriAI/litellm/pull/20653) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260207110613_add_soft_budget_to_deleted_teams_table/migration.sql) |
| `LiteLLM_MCPServerTable` | New Column | Added `available_on_public_internet` boolean for MCP IP-based access control | [PR #20607](https://github.com/BerriAI/litellm/pull/20607) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260207093506_add_available_on_public_internet_to_mcp_servers/migration.sql) |
---
## Documentation Updates (14 updates)
- Add FAQ for setting up and verifying LITELLM_LICENSE - [PR #20284](https://github.com/BerriAI/litellm/pull/20284)
- Model request tags documentation - [PR #20290](https://github.com/BerriAI/litellm/pull/20290)
- Add Prisma migration troubleshooting guide - [PR #20300](https://github.com/BerriAI/litellm/pull/20300)
- MCP Semantic Filtering documentation - [PR #20316](https://github.com/BerriAI/litellm/pull/20316)
- Add CopilotKit SDK doc as supported agents SDK - [PR #20396](https://github.com/BerriAI/litellm/pull/20396)
- Add documentation for Nova Sonic - [PR #20320](https://github.com/BerriAI/litellm/pull/20320)
- Update Vertex AI Text to Speech doc to show use of audio - [PR #20255](https://github.com/BerriAI/litellm/pull/20255)
- Improve Okta SSO setup guide with step-by-step instructions - [PR #20353](https://github.com/BerriAI/litellm/pull/20353)
- Langfuse doc update - [PR #20443](https://github.com/BerriAI/litellm/pull/20443)
- Expose MCPs on public internet documentation - [PR #20626](https://github.com/BerriAI/litellm/pull/20626)
- Add blog post: Achieving Sub-Millisecond Proxy Overhead - [PR #20309](https://github.com/BerriAI/litellm/pull/20309)
- Add blog post about litellm-observatory - [PR #20622](https://github.com/BerriAI/litellm/pull/20622)
- Update Opus 4.6 blog with adaptive thinking - [PR #20637](https://github.com/BerriAI/litellm/pull/20637)
- `gpt-5-search-api` docs clarifications - [PR #20512](https://github.com/BerriAI/litellm/pull/20512)
---
## New Contributors
* @Quentin-M made their first contribution in [PR #19818](https://github.com/BerriAI/litellm/pull/19818)
* @amirzaushnizer made their first contribution in [PR #20235](https://github.com/BerriAI/litellm/pull/20235)
* @cscguochang made their first contribution in [PR #20214](https://github.com/BerriAI/litellm/pull/20214)
* @krauckbot made their first contribution in [PR #20273](https://github.com/BerriAI/litellm/pull/20273)
* @agrattan0820 made their first contribution in [PR #19784](https://github.com/BerriAI/litellm/pull/19784)
* @nina-hu made their first contribution in [PR #20472](https://github.com/BerriAI/litellm/pull/20472)
* @swayambhu94 made their first contribution in [PR #20469](https://github.com/BerriAI/litellm/pull/20469)
* @ssadedin made their first contribution in [PR #20566](https://github.com/BerriAI/litellm/pull/20566)
---
## Full Changelog
[v1.81.6-nightly...v1.81.9](https://github.com/BerriAI/litellm/compare/v1.81.6-nightly...v1.81.9)

43
docs/my-website/sidebars.js
View file

@ -134,6 +134,7 @@ const sidebars = {
"tutorials/claude_mcp",
"tutorials/claude_non_anthropic_models",
"tutorials/claude_code_plugin_marketplace",
"tutorials/claude_code_beta_headers",
]
},
"tutorials/opencode_integration",
@ -291,40 +292,52 @@ const sidebars = {
label: "Admin UI",
items: [
"proxy/ui",
"proxy/admin_ui_sso",
"proxy/custom_root_ui",
"proxy/custom_sso",
"proxy/ai_hub",
"proxy/model_compare_ui",
"proxy/ui_credentials",
"tutorials/scim_litellm",
{
type: "category",
label: "UI User/Team Management",
label: "Setup & SSO",
items: [
"proxy/access_control",
"proxy/public_teams",
"proxy/admin_ui_sso",
"proxy/custom_sso",
"proxy/custom_root_ui",
"tutorials/scim_litellm",
]
},
{
type: "category",
label: "Models",
items: [
"proxy/ui_credentials",
"proxy/ai_hub",
"proxy/model_compare_ui",
]
},
{
type: "category",
label: "Teams & Organizations",
items: [
"proxy/access_control",
"proxy/self_serve",
"proxy/public_teams",
"proxy/ui/bulk_edit_users",
"proxy/ui/page_visibility",
]
},
{
type: "category",
label: "UI Usage Tracking",
label: "Observability: Usage",
items: [
"proxy/customer_usage",
"proxy/endpoint_activity"
"proxy/endpoint_activity",
]
},
{
type: "category",
label: "UI Logs",
label: "Logs",
items: [
"proxy/ui_logs",
"proxy/ui_spend_log_settings",
"proxy/ui_logs_sessions",
"proxy/deleted_keys_teams"
"proxy/deleted_keys_teams",
]
}
],
@ -372,6 +385,7 @@ const sidebars = {
items: [
"proxy/users",
"proxy/team_budgets",
"proxy/ui_team_soft_budget_alerts",
"proxy/tag_budgets",
"proxy/customers",
"proxy/dynamic_rate_limit",
@ -547,6 +561,7 @@ const sidebars = {
items: [
"mcp",
"mcp_usage",
"mcp_public_internet",
"mcp_semantic_filter",
"mcp_control",
"mcp_cost",

133
docs/my-website/src/components/MiddlewareDiagrams/BaseHTTPMiddlewareAnimation.tsx Normal file
View file

@ -0,0 +1,133 @@
import React, { useState, useEffect, useCallback, useRef } from 'react';
import styles from './styles.module.css';
interface Stage {
label: string;
subtitle: string;
code: string;
}
const STAGES: Stage[] = [
{
label: 'Request Wrapping',
subtitle: '_CachedRequest',
code: 'request = _CachedRequest(scope, receive)',
},
{
label: 'Sync Event',
subtitle: 'anyio.Event()',
code: 'response_sent = anyio.Event()',
},
{
label: 'Memory Stream',
subtitle: 'create_memory_object_stream()',
code: 'send_stream, recv_stream = anyio.create_memory_object_stream()',
},
{
label: 'Task Group',
subtitle: 'create_task_group()',
code: 'async with anyio.create_task_group() as task_group:',
},
{
label: 'Background Task',
subtitle: 'task_group.start_soon(coro)',
code: 'task_group.start_soon(coro) # app runs in separate task',
},
{
label: 'Nested Task Group',
subtitle: 'receive_or_disconnect()',
code: 'async with anyio.create_task_group() as task_group: ...',
},
{
label: 'Response Wrapping',
subtitle: '_StreamingResponse',
code: 'response = _StreamingResponse(status_code=..., content=body_stream())',
},
];
const INTERVAL_MS = 1200;
const PAUSE_MS = 600;
export default function BaseHTTPMiddlewareAnimation() {
const [activeStage, setActiveStage] = useState(0);
const [paused, setPaused] = useState(false);
const [expandedStage, setExpandedStage] = useState<number | null>(null);
const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const clearTimer = useCallback(() => {
if (timerRef.current !== null) {
clearTimeout(timerRef.current);
timerRef.current = null;
}
}, []);
useEffect(() => {
if (paused) return;
const advance = () => {
setActiveStage((prev) => {
const next = (prev + 1) % STAGES.length;
// If wrapping around, add extra pause
if (next === 0) {
timerRef.current = setTimeout(() => {
timerRef.current = setTimeout(advance, INTERVAL_MS);
}, PAUSE_MS);
return next;
}
timerRef.current = setTimeout(advance, INTERVAL_MS);
return next;
});
};
timerRef.current = setTimeout(advance, INTERVAL_MS);
return clearTimer;
}, [paused, clearTimer]);
const handleStageClick = (index: number) => {
clearTimer();
setPaused(true);
setActiveStage(index);
if (expandedStage === index) {
// Close panel and resume
setExpandedStage(null);
setPaused(false);
} else {
setExpandedStage(index);
}
};
return (
<div className={styles.pipelineWrapper}>
<div className={styles.pipelineLabel}>7 steps per request</div>
<div className={styles.pipeline}>
{STAGES.map((stage, i) => (
<div className={styles.stageWrapper} key={i}>
<div
className={`${styles.stage} ${activeStage === i ? styles.stageActive : ''}`}
onClick={() => handleStageClick(i)}
role="button"
tabIndex={0}
onKeyDown={(e) => {
if (e.key === 'Enter' || e.key === ' ') handleStageClick(i);
}}
>
<div className={styles.stageNumber}>{i + 1}</div>
<div className={styles.stageLabel}>{stage.label}</div>
<div className={styles.stageSubtitle}>{stage.subtitle}</div>
</div>
</div>
))}
</div>
<div
className={`${styles.codePanel} ${expandedStage !== null ? styles.codePanelOpen : ''}`}
>
{expandedStage !== null && (
<pre className={styles.codePanelCode}>
<code>{STAGES[expandedStage].code}</code>
</pre>
)}
</div>
</div>
);
}

337
docs/my-website/src/components/MiddlewareDiagrams/BenchmarkVisualization.tsx Normal file
View file

@ -0,0 +1,337 @@
import React, { useState, useEffect, useRef, useCallback } from 'react';
import styles from './styles.module.css';
/* ── Constants ── */
const TOTAL_REQUESTS = 50_000;
const DURATION_AFTER_MS = 8_000; // "After" column finishes in 8s
const DURATION_BEFORE_MS = 13_920; // 74% slower → 8000 * 1.74
const TICK_MS = 50;
const RESET_PAUSE_MS = 2_000;
const MAX_DOTS = 14;
const BEFORE_RPS = 3_785;
const AFTER_RPS = 6_577;
const BEFORE_P50 = 21;
const AFTER_P50 = 13;
const BEFORE_LAYERS = [
{ label: 'ab client', warning: false },
{ label: 'uvicorn \u00B7 1 worker', warning: false },
{ label: 'ASGI Middleware', warning: false },
{ label: 'BaseHTTPMiddleware', warning: true },
{ label: 'GET /health \u2192 "ok"', warning: false },
];
const AFTER_LAYERS = [
{ label: 'ab client', warning: false },
{ label: 'uvicorn \u00B7 1 worker', warning: false },
{ label: 'ASGI Middleware', warning: false },
{ label: 'ASGI Middleware', warning: false },
{ label: 'GET /health \u2192 "ok"', warning: false },
];
const BENCHMARK_RUNS = [
{ config: 'Before (1 ASGI + 1 BaseHTTP)', run: 1, rps: 3596, p50: 21 },
{ config: 'Before (1 ASGI + 1 BaseHTTP)', run: 2, rps: 3599, p50: 21 },
{ config: 'Before (1 ASGI + 1 BaseHTTP)', run: 3, rps: 4161, p50: 21 },
{ config: 'After (2x Pure ASGI)', run: 1, rps: 6504, p50: 13 },
{ config: 'After (2x Pure ASGI)', run: 2, rps: 6631, p50: 13 },
{ config: 'After (2x Pure ASGI)', run: 3, rps: 6595, p50: 13 },
];
/* ── Dot type ── */
interface Dot {
id: number;
progress: number; // 0..1 (top to bottom)
}
/* ── Component ── */
export default function BenchmarkVisualization() {
const [elapsed, setElapsed] = useState(0);
const [running, setRunning] = useState(false);
const [afterDone, setAfterDone] = useState(false);
const [beforeDone, setBeforeDone] = useState(false);
const [tableOpen, setTableOpen] = useState(false);
const [beforeDots, setBeforeDots] = useState<Dot[]>([]);
const [afterDots, setAfterDots] = useState<Dot[]>([]);
const dotIdRef = useRef(0);
const observerRef = useRef<IntersectionObserver | null>(null);
const wrapperRef = useRef<HTMLDivElement | null>(null);
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
const hasStartedRef = useRef(false);
const beforeProgress = Math.min(elapsed / DURATION_BEFORE_MS, 1);
const afterProgress = Math.min(elapsed / DURATION_AFTER_MS, 1);
const beforeCompleted = Math.round(beforeProgress * TOTAL_REQUESTS);
const afterCompleted = Math.round(afterProgress * TOTAL_REQUESTS);
const beforeCurrentRPS = running && !beforeDone
? Math.round(BEFORE_RPS * (0.9 + Math.random() * 0.2))
: beforeDone ? 0 : 0;
const afterCurrentRPS = running && !afterDone
? Math.round(AFTER_RPS * (0.9 + Math.random() * 0.2))
: afterDone ? 0 : 0;
const reset = useCallback(() => {
setElapsed(0);
setAfterDone(false);
setBeforeDone(false);
setBeforeDots([]);
setAfterDots([]);
dotIdRef.current = 0;
}, []);
// Start/restart loop
const startSimulation = useCallback(() => {
reset();
setRunning(true);
}, [reset]);
// IntersectionObserver to auto-start on scroll
useEffect(() => {
observerRef.current = new IntersectionObserver(
([entry]) => {
if (entry.isIntersecting && !hasStartedRef.current) {
hasStartedRef.current = true;
startSimulation();
}
},
{ threshold: 0.3 }
);
if (wrapperRef.current) {
observerRef.current.observe(wrapperRef.current);
}
return () => {
observerRef.current?.disconnect();
};
}, [startSimulation]);
// Main tick
useEffect(() => {
if (!running) return;
timerRef.current = setInterval(() => {
setElapsed((prev) => {
const next = prev + TICK_MS;
if (next >= DURATION_AFTER_MS) setAfterDone(true);
if (next >= DURATION_BEFORE_MS) setBeforeDone(true);
// Both done → schedule reset
if (next >= DURATION_BEFORE_MS) {
setTimeout(() => {
startSimulation();
}, RESET_PAUSE_MS);
setRunning(false);
return next;
}
return next;
});
}, TICK_MS);
return () => {
if (timerRef.current) clearInterval(timerRef.current);
};
}, [running, startSimulation]);
// Dot animation
useEffect(() => {
if (!running) return;
const dotInterval = setInterval(() => {
const spawnBefore = !beforeDone && Math.random() < 0.4;
const spawnAfter = !afterDone && Math.random() < 0.65;
if (spawnBefore) {
setBeforeDots((prev) => {
const dots = [...prev, { id: dotIdRef.current++, progress: 0 }];
return dots.slice(-MAX_DOTS);
});
}
if (spawnAfter) {
setAfterDots((prev) => {
const dots = [...prev, { id: dotIdRef.current++, progress: 0 }];
return dots.slice(-MAX_DOTS);
});
}
// Advance existing dots
setBeforeDots((prev) =>
prev
.map((d) => ({ ...d, progress: d.progress + 0.08 }))
.filter((d) => d.progress <= 1)
);
setAfterDots((prev) =>
prev
.map((d) => ({ ...d, progress: d.progress + 0.14 }))
.filter((d) => d.progress <= 1)
);
}, 100);
return () => clearInterval(dotInterval);
}, [running, beforeDone, afterDone]);
const renderFlowStack = (
layers: { label: string; warning: boolean }[],
dots: Dot[],
isBefore: boolean
) => (
<div className={styles.flowStack}>
<div className={styles.dotsCanvas}>
{dots.map((dot) => (
<div
key={dot.id}
className={`${styles.dot} ${isBefore ? styles.dotSlow : styles.dotFast}`}
style={{
top: `${dot.progress * 92}%`,
left: `${48 + Math.sin(dot.id * 1.7) * 12}%`,
opacity: dot.progress > 0.85 ? (1 - dot.progress) * 6 : 0.8,
}}
/>
))}
</div>
{layers.map((layer, i) => (
<React.Fragment key={i}>
{i > 0 && <div className={styles.flowArrow}>&darr;</div>}
<div
className={`${styles.flowLayer} ${layer.warning ? styles.flowLayerWarning : ''}`}
>
{layer.label}
{layer.warning && <span className={styles.overheadTag}>&larr; overhead</span>}
</div>
</React.Fragment>
))}
</div>
);
const formatNum = (n: number) => n.toLocaleString();
return (
<div className={styles.benchmarkWrapper} ref={wrapperRef}>
<div className={styles.benchmarkConfig}>
50,000 requests &middot; 1,000 concurrent &middot; 1 worker
</div>
<div className={styles.benchmarkColumns}>
{/* Before column */}
<div className={styles.benchmarkColumn}>
<div className={`${styles.columnTitle} ${styles.columnTitleBefore}`}>
Before (1 ASGI + 1 BaseHTTP)
{beforeDone && (
<span className={`${styles.doneBadge} ${styles.doneBadgeBefore}`}>done</span>
)}
</div>
{renderFlowStack(BEFORE_LAYERS, beforeDots, true)}
<div className={styles.statsRow}>
<div className={styles.stat}>
<div className={styles.statValue}>{formatNum(beforeCurrentRPS)}</div>
<div className={styles.statLabel}>RPS</div>
</div>
<div className={styles.stat}>
<div className={styles.statValue}>{formatNum(beforeCompleted)}</div>
<div className={styles.statLabel}>Completed</div>
</div>
<div className={styles.stat}>
<div className={styles.statValue}>{BEFORE_P50}ms</div>
<div className={styles.statLabel}>P50</div>
</div>
</div>
<div className={styles.progressBar}>
<div
className={`${styles.progressFill} ${styles.progressFillBefore}`}
style={{ width: `${beforeProgress * 100}%` }}
/>
</div>
</div>
{/* After column */}
<div className={styles.benchmarkColumn}>
<div className={`${styles.columnTitle} ${styles.columnTitleAfter}`}>
After (2x Pure ASGI)
{afterDone && (
<span className={`${styles.doneBadge} ${styles.doneBadgeAfter}`}>done</span>
)}
</div>
{renderFlowStack(AFTER_LAYERS, afterDots, false)}
<div className={styles.statsRow}>
<div className={styles.stat}>
<div className={styles.statValue}>{formatNum(afterCurrentRPS)}</div>
<div className={styles.statLabel}>RPS</div>
</div>
<div className={styles.stat}>
<div className={styles.statValue}>{formatNum(afterCompleted)}</div>
<div className={styles.statLabel}>Completed</div>
</div>
<div className={styles.stat}>
<div className={styles.statValue}>{AFTER_P50}ms</div>
<div className={styles.statLabel}>P50</div>
</div>
</div>
<div className={styles.progressBar}>
<div
className={`${styles.progressFill} ${styles.progressFillAfter}`}
style={{ width: `${afterProgress * 100}%` }}
/>
</div>
</div>
</div>
{/* Summary stats */}
<div className={styles.summaryStats}>
<div className={styles.summaryItem}>
<div className={styles.summaryValue}>+74%</div>
<div className={styles.summaryLabel}>Throughput (RPS)</div>
</div>
<div className={styles.summaryItem}>
<div className={styles.summaryValue}>-38%</div>
<div className={styles.summaryLabel}>Median Latency (P50)</div>
</div>
</div>
{/* Collapsible per-run data */}
<div className={styles.collapsible}>
<button
className={styles.collapsibleToggle}
onClick={() => setTableOpen(!tableOpen)}
>
<span
className={`${styles.collapsibleChevron} ${
tableOpen ? styles.collapsibleChevronOpen : ''
}`}
>
&#9654;
</span>
Per-run data (3 runs each)
</button>
<div
className={`${styles.collapsibleContent} ${
tableOpen ? styles.collapsibleContentOpen : ''
}`}
>
<table className={styles.dataTable}>
<thead>
<tr>
<th>Config</th>
<th>Run</th>
<th>RPS</th>
<th>P50 (ms)</th>
</tr>
</thead>
<tbody>
{BENCHMARK_RUNS.map((row, i) => (
<tr key={i}>
<td>{row.config}</td>
<td>{row.run}</td>
<td>{formatNum(row.rps)}</td>
<td>{row.p50}</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
</div>
);
}

67
docs/my-website/src/components/MiddlewareDiagrams/PureASGIAnimation.tsx Normal file
View file

@ -0,0 +1,67 @@
import React, { useState, useEffect, useRef, useCallback } from 'react';
import styles from './styles.module.css';
interface Stage {
label: string;
subtitle: string;
}
const STAGES: Stage[] = [
{ label: 'Scope Check', subtitle: 'scope["type"] != "http"' },
{ label: 'Direct Call', subtitle: 'await self.app(scope, receive, send)' },
];
const INTERVAL_MS = 1200;
const PAUSE_MS = 600;
export default function PureASGIAnimation() {
const [activeStage, setActiveStage] = useState(0);
const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const clearTimer = useCallback(() => {
if (timerRef.current !== null) {
clearTimeout(timerRef.current);
timerRef.current = null;
}
}, []);
useEffect(() => {
const advance = () => {
setActiveStage((prev) => {
const next = (prev + 1) % STAGES.length;
if (next === 0) {
timerRef.current = setTimeout(() => {
timerRef.current = setTimeout(advance, INTERVAL_MS);
}, PAUSE_MS);
return next;
}
timerRef.current = setTimeout(advance, INTERVAL_MS);
return next;
});
};
timerRef.current = setTimeout(advance, INTERVAL_MS);
return clearTimer;
}, [clearTimer]);
return (
<div className={styles.pipelineWrapper}>
<div className={styles.pipelineLabel}>2 steps per request</div>
<div className={`${styles.pipeline} ${styles.pipelineTwoCol}`}>
{STAGES.map((stage, i) => (
<div className={styles.stageWrapper} key={i}>
<div
className={`${styles.stage} ${styles.stageNoClick} ${
activeStage === i ? styles.stageActiveGreen : ''
}`}
>
<div className={styles.stageNumber}>{i + 1}</div>
<div className={styles.stageLabel}>{stage.label}</div>
<div className={styles.stageSubtitle}>{stage.subtitle}</div>
</div>
</div>
))}
</div>
</div>
);
}

3
docs/my-website/src/components/MiddlewareDiagrams/index.tsx Normal file
View file

@ -0,0 +1,3 @@
export { default as BaseHTTPMiddlewareAnimation } from './BaseHTTPMiddlewareAnimation';
export { default as PureASGIAnimation } from './PureASGIAnimation';
export { default as BenchmarkVisualization } from './BenchmarkVisualization';

494
docs/my-website/src/components/MiddlewareDiagrams/styles.module.css Normal file
View file

@ -0,0 +1,494 @@
/* ── Shared custom properties ── */
:root {
--mw-stage-bg: #f8f9fa;
--mw-stage-border: #dee2e6;
--mw-stage-active-bg: #e8f4fd;
--mw-stage-active-border: #3b82f6;
--mw-stage-green-active-bg: #ecfdf5;
--mw-stage-green-active-border: #10b981;
--mw-dot-color: #3b82f6;
--mw-warning-accent: #ef4444;
--mw-success-accent: #10b981;
--mw-text-primary: #1a1a2e;
--mw-text-secondary: #6b7280;
--mw-code-bg: #f1f5f9;
--mw-panel-bg: #ffffff;
--mw-panel-border: #e5e7eb;
--mw-bar-bg: #e5e7eb;
--mw-arrow-color: #9ca3af;
--mw-column-bg: #fafafa;
--mw-column-border: #e5e7eb;
--mw-layer-bg: #f3f4f6;
--mw-layer-border: #d1d5db;
--mw-layer-warning-bg: #fef2f2;
--mw-layer-warning-border: #fca5a5;
--mw-progress-bg: #e5e7eb;
}
[data-theme='dark'] {
--mw-stage-bg: #1e1e2e;
--mw-stage-border: #374151;
--mw-stage-active-bg: #1e3a5f;
--mw-stage-active-border: #60a5fa;
--mw-stage-green-active-bg: #064e3b;
--mw-stage-green-active-border: #34d399;
--mw-dot-color: #60a5fa;
--mw-warning-accent: #f87171;
--mw-success-accent: #34d399;
--mw-text-primary: #e5e7eb;
--mw-text-secondary: #9ca3af;
--mw-code-bg: #1e293b;
--mw-panel-bg: #111827;
--mw-panel-border: #374151;
--mw-bar-bg: #374151;
--mw-arrow-color: #6b7280;
--mw-column-bg: #111827;
--mw-column-border: #374151;
--mw-layer-bg: #1f2937;
--mw-layer-border: #4b5563;
--mw-layer-warning-bg: #451a1a;
--mw-layer-warning-border: #b91c1c;
--mw-progress-bg: #374151;
}
/* ── Pipeline (shared between BaseHTTP and PureASGI) ── */
.pipelineWrapper {
margin: 1.5rem 0;
}
.pipelineLabel {
text-align: center;
font-size: 0.85rem;
font-weight: 600;
color: var(--mw-text-secondary);
margin-bottom: 0.75rem;
text-transform: uppercase;
letter-spacing: 0.05em;
}
.pipeline {
display: flex;
flex-wrap: wrap;
justify-content: center;
align-items: stretch;
gap: 0.75rem;
padding: 0.5rem 0;
}
.pipelineTwoCol {
max-width: 480px;
margin: 0 auto;
}
.stageWrapper {
display: flex;
align-items: center;
width: 160px;
flex-shrink: 0;
}
.pipelineTwoCol .stageWrapper {
width: 200px;
}
.arrow {
display: none;
}
.stage {
flex: 1;
padding: 0.85rem 0.75rem;
min-height: 100px;
display: flex;
flex-direction: column;
justify-content: center;
background: var(--mw-stage-bg);
border: 2px solid var(--mw-stage-border);
border-radius: 8px;
text-align: center;
cursor: pointer;
transition: background 0.4s ease, border-color 0.4s ease, box-shadow 0.4s ease;
user-select: none;
}
.stage:hover {
border-color: var(--mw-stage-active-border);
}
.stageActive {
background: var(--mw-stage-active-bg);
border-color: var(--mw-stage-active-border);
box-shadow: 0 0 0 3px rgba(59, 130, 246, 0.15);
}
.stageActiveGreen {
background: var(--mw-stage-green-active-bg);
border-color: var(--mw-stage-green-active-border);
box-shadow: 0 0 0 3px rgba(16, 185, 129, 0.15);
}
.stageNoClick {
cursor: default;
}
.stageNumber {
font-size: 0.7rem;
font-weight: 700;
color: var(--mw-text-secondary);
margin-bottom: 0.3rem;
}
.stageLabel {
font-size: 0.85rem;
font-weight: 600;
color: var(--mw-text-primary);
margin-bottom: 0.25rem;
line-height: 1.3;
}
.stageSubtitle {
font-size: 0.72rem;
color: var(--mw-text-secondary);
font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
word-break: break-word;
line-height: 1.3;
}
/* ── Code panel (accordion) ── */
.codePanel {
max-height: 0;
overflow: hidden;
transition: max-height 0.35s ease, padding 0.35s ease;
background: var(--mw-code-bg);
border-radius: 0 0 8px 8px;
margin-top: 0.5rem;
}
.codePanelOpen {
max-height: 120px;
padding: 0.75rem 1rem;
}
.codePanelCode {
font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
font-size: 0.8rem;
color: var(--mw-text-primary);
white-space: pre;
margin: 0;
line-height: 1.5;
}
/* ── Benchmark Visualization ── */
.benchmarkWrapper {
margin: 1.5rem 0;
}
.benchmarkConfig {
text-align: center;
font-size: 0.85rem;
color: var(--mw-text-secondary);
margin-bottom: 1rem;
font-weight: 500;
}
.benchmarkColumns {
display: flex;
gap: 1.5rem;
}
.benchmarkColumn {
flex: 1;
background: var(--mw-column-bg);
border: 1px solid var(--mw-column-border);
border-radius: 12px;
padding: 1.25rem;
position: relative;
overflow: hidden;
}
.columnTitle {
font-size: 0.9rem;
font-weight: 700;
color: var(--mw-text-primary);
text-align: center;
margin-bottom: 1rem;
}
.columnTitleBefore {
color: var(--mw-warning-accent);
}
.columnTitleAfter {
color: var(--mw-success-accent);
}
/* ── Request flow stack ── */
.flowStack {
display: flex;
flex-direction: column;
align-items: center;
gap: 0;
position: relative;
min-height: 280px;
}
.flowLayer {
width: 100%;
max-width: 260px;
padding: 0.6rem 0.75rem;
background: var(--mw-layer-bg);
border: 1px solid var(--mw-layer-border);
border-radius: 6px;
text-align: center;
font-size: 0.78rem;
font-weight: 500;
color: var(--mw-text-primary);
position: relative;
z-index: 1;
}
.flowLayerWarning {
background: var(--mw-layer-warning-bg);
border-color: var(--mw-layer-warning-border);
font-weight: 700;
}
.flowArrow {
display: flex;
justify-content: center;
color: var(--mw-arrow-color);
font-size: 0.9rem;
padding: 0.15rem 0;
position: relative;
z-index: 0;
min-height: 20px;
}
.overheadTag {
font-size: 0.65rem;
color: var(--mw-warning-accent);
margin-left: 0.4rem;
}
/* ── Dots layer (canvas for flowing dots) ── */
.dotsCanvas {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
pointer-events: none;
z-index: 2;
}
.dot {
position: absolute;
width: 6px;
height: 6px;
border-radius: 50%;
background: var(--mw-dot-color);
opacity: 0.8;
}
.dotSlow {
background: var(--mw-warning-accent);
}
.dotFast {
background: var(--mw-success-accent);
}
/* ── Stats & progress ── */
.statsRow {
display: flex;
justify-content: space-around;
margin-top: 1rem;
padding-top: 0.75rem;
border-top: 1px solid var(--mw-panel-border);
}
.stat {
text-align: center;
}
.statValue {
font-size: 1.1rem;
font-weight: 700;
color: var(--mw-text-primary);
font-variant-numeric: tabular-nums;
}
.statLabel {
font-size: 0.7rem;
color: var(--mw-text-secondary);
text-transform: uppercase;
letter-spacing: 0.04em;
}
.progressBar {
width: 100%;
height: 6px;
background: var(--mw-progress-bg);
border-radius: 3px;
margin-top: 0.75rem;
overflow: hidden;
}
.progressFill {
height: 100%;
border-radius: 3px;
transition: width 0.1s linear;
}
.progressFillBefore {
background: var(--mw-warning-accent);
}
.progressFillAfter {
background: var(--mw-success-accent);
}
/* ── Summary stats below simulation ── */
.summaryStats {
display: flex;
justify-content: center;
gap: 2rem;
margin-top: 1.5rem;
flex-wrap: wrap;
}
.summaryItem {
text-align: center;
padding: 0.75rem 1.25rem;
background: var(--mw-stage-bg);
border-radius: 8px;
border: 1px solid var(--mw-panel-border);
}
.summaryValue {
font-size: 1.5rem;
font-weight: 800;
color: var(--mw-success-accent);
}
.summaryLabel {
font-size: 0.8rem;
color: var(--mw-text-secondary);
margin-top: 0.2rem;
}
/* ── Collapsible table ── */
.collapsible {
margin-top: 1.5rem;
}
.collapsibleToggle {
background: none;
border: 1px solid var(--mw-panel-border);
border-radius: 6px;
padding: 0.5rem 1rem;
cursor: pointer;
font-size: 0.85rem;
color: var(--mw-text-primary);
width: 100%;
text-align: left;
display: flex;
align-items: center;
gap: 0.5rem;
transition: background 0.2s;
}
.collapsibleToggle:hover {
background: var(--mw-stage-bg);
}
.collapsibleChevron {
transition: transform 0.3s ease;
font-size: 0.7rem;
}
.collapsibleChevronOpen {
transform: rotate(90deg);
}
.collapsibleContent {
max-height: 0;
overflow: hidden;
transition: max-height 0.35s ease;
}
.collapsibleContentOpen {
max-height: 600px;
}
.dataTable {
width: 100%;
border-collapse: collapse;
margin-top: 0.75rem;
font-size: 0.85rem;
}
.dataTable th,
.dataTable td {
padding: 0.5rem 0.75rem;
text-align: left;
border-bottom: 1px solid var(--mw-panel-border);
}
.dataTable th {
font-weight: 600;
color: var(--mw-text-secondary);
font-size: 0.75rem;
text-transform: uppercase;
letter-spacing: 0.04em;
}
.dataTable td {
color: var(--mw-text-primary);
font-variant-numeric: tabular-nums;
}
/* ── Reproduce section ── */
.reproduceSection {
margin-top: 1rem;
}
/* ── Done badge ── */
.doneBadge {
display: inline-block;
font-size: 0.75rem;
font-weight: 600;
padding: 0.2rem 0.6rem;
border-radius: 4px;
margin-left: 0.5rem;
}
.doneBadgeBefore {
color: var(--mw-warning-accent);
background: var(--mw-layer-warning-bg);
}
.doneBadgeAfter {
color: var(--mw-success-accent);
background: var(--mw-stage-green-active-bg);
}
/* ── Responsive ── */
@media (max-width: 768px) {
.stageWrapper {
width: 140px;
}
.pipelineTwoCol .stageWrapper {
width: 160px;
}
.benchmarkColumns {
flex-direction: column;
}
.summaryStats {
flex-direction: column;
align-items: center;
}
}

BIN
enterprise/dist/litellm_enterprise-0.1.30-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.30.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.31-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.31.tar.gz vendored Normal file
View file

Binary file not shown.

119
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py
View file

@ -30,8 +30,15 @@ from litellm.integrations.email_templates.user_invitation_email import (
from litellm.integrations.email_templates.templates import (
MAX_BUDGET_ALERT_EMAIL_TEMPLATE,
SOFT_BUDGET_ALERT_EMAIL_TEMPLATE,
TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE,
)
from litellm.proxy._types import (
CallInfo,
InvitationNew,
Litellm_EntityType,
UserAPIKeyAuth,
WebhookEvent,
)
from litellm.proxy._types import CallInfo, InvitationNew, UserAPIKeyAuth, WebhookEvent
from litellm.secret_managers.main import get_secret_bool
from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL
from litellm.constants import (
@ -217,6 +224,78 @@ class BaseEmailLogger(CustomLogger):
)
pass
async def send_team_soft_budget_alert_email(self, event: WebhookEvent):
"""
Send email to team members when team soft budget is crossed
Supports multiple recipients via alert_emails field from team metadata
"""
# Collect all recipient emails
recipient_emails: List[str] = []
# Add additional alert emails from team metadata.soft_budget_alert_emails
if hasattr(event, "alert_emails") and event.alert_emails:
for email in event.alert_emails:
if email and email not in recipient_emails: # Avoid duplicates
recipient_emails.append(email)
# If no recipients found, skip sending
if not recipient_emails:
verbose_proxy_logger.warning(
f"No recipient emails found for team soft budget alert. event={event.model_dump(exclude_none=True)}"
)
return
# Validate that we have at least one valid email address
first_recipient_email = recipient_emails[0]
if not first_recipient_email or not first_recipient_email.strip():
verbose_proxy_logger.warning(
f"Invalid recipient email found for team soft budget alert. event={event.model_dump(exclude_none=True)}"
)
return
verbose_proxy_logger.debug(
f"send_team_soft_budget_alert_email_event: {json.dumps(event.model_dump(exclude_none=True), indent=4, default=str)}"
)
# Get email params using the first recipient email (for template formatting)
# For team alerts with alert_emails, we don't need user_id lookup since we already have email addresses
# Pass user_id=None to prevent _get_email_params from trying to look up email from a potentially None user_id
email_params = await self._get_email_params(
email_event=EmailEvent.soft_budget_crossed,
user_id=None, # Team alerts don't require user_id when alert_emails are provided
user_email=first_recipient_email,
event_message=event.event_message,
)
# Format budget values
soft_budget_str = f"${event.soft_budget}" if event.soft_budget is not None else "N/A"
spend_str = f"${event.spend}" if event.spend is not None else "$0.00"
max_budget_info = ""
if event.max_budget is not None:
max_budget_info = f"<b>Maximum Budget:</b> ${event.max_budget} <br />"
# Use team alias or generic greeting
team_alias = event.team_alias or "Team"
email_html_content = TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE.format(
email_logo_url=email_params.logo_url,
team_alias=team_alias,
soft_budget=soft_budget_str,
spend=spend_str,
max_budget_info=max_budget_info,
base_url=email_params.base_url,
email_support_contact=email_params.support_contact,
)
# Send email to all recipients
await self.send_email(
from_email=self.DEFAULT_LITELLM_EMAIL,
to_email=recipient_emails,
subject=email_params.subject,
html_body=email_html_content,
)
pass
async def send_max_budget_alert_email(self, event: WebhookEvent):
"""
Send email to user when max budget alert threshold is reached
@ -285,15 +364,36 @@ class BaseEmailLogger(CustomLogger):
# - Don't re-alert, if alert already sent
_cache: DualCache = self.internal_usage_cache
# percent of max_budget left to spend
if user_info.max_budget is None and user_info.soft_budget is None:
return
# For soft_budget alerts, check if we've already sent an alert
if type == "soft_budget":
# For team soft budget alerts, we only need team soft_budget to be set
# For other entity types, we need either max_budget or soft_budget
if user_info.event_group == Litellm_EntityType.TEAM:
if user_info.soft_budget is None:
return
# For team soft budget alerts, require alert_emails to be configured
# Team soft budget alerts are sent via metadata.soft_budget_alerting_emails
if user_info.alert_emails is None or len(user_info.alert_emails) == 0:
verbose_proxy_logger.debug(
"Skipping team soft budget email alert: no alert_emails configured",
)
return
else:
# For non-team alerts, require either max_budget or soft_budget
if user_info.max_budget is None and user_info.soft_budget is None:
return
if user_info.soft_budget is not None and user_info.spend >= user_info.soft_budget:
# Generate cache key based on event type and identifier
_id = user_info.token or user_info.user_id or "default_id"
# Use appropriate ID based on event_group to ensure unique cache keys per entity type
if user_info.event_group == Litellm_EntityType.TEAM:
_id = user_info.team_id or "default_id"
elif user_info.event_group == Litellm_EntityType.ORGANIZATION:
_id = user_info.organization_id or "default_id"
elif user_info.event_group == Litellm_EntityType.USER:
_id = user_info.user_id or "default_id"
else:
# For KEY and other types, use token or user_id
_id = user_info.token or user_info.user_id or "default_id"
_cache_key = f"email_budget_alerts:soft_budget_crossed:{_id}"
# Check if we've already sent this alert
@ -318,10 +418,15 @@ class BaseEmailLogger(CustomLogger):
projected_exceeded_date=user_info.projected_exceeded_date,
projected_spend=user_info.projected_spend,
event_group=user_info.event_group,
alert_emails=user_info.alert_emails,
)
try:
await self.send_soft_budget_alert_email(webhook_event)
# Use team-specific function for team alerts, otherwise use standard function
if user_info.event_group == Litellm_EntityType.TEAM:
await self.send_team_soft_budget_alert_email(webhook_event)
else:
await self.send_soft_budget_alert_email(webhook_event)
# Cache the alert to prevent duplicate sends
await _cache.async_set_cache(

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.29"
version = "0.1.31"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.29"
version = "0.1.31"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.31-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.31.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.33-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.33.tar.gz vendored Normal file
View file

Binary file not shown.

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260205144610_add_soft_budget_to_team_table/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN "soft_budget" DOUBLE PRECISION;

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260207093506_add_available_on_public_internet_to_mcp_servers/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "available_on_public_internet" BOOLEAN NOT NULL DEFAULT false;

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260207110613_add_soft_budget_to_deleted_teams_table/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN "soft_budget" DOUBLE PRECISION;

8
litellm-proxy-extras/litellm_proxy_extras/migrations/20260209085821_add_verificationtoken_indexes/migration.sql Normal file
View file

@ -0,0 +1,8 @@
-- CreateIndex
CREATE INDEX "LiteLLM_VerificationToken_user_id_team_id_idx" ON "LiteLLM_VerificationToken"("user_id", "team_id");
-- CreateIndex
CREATE INDEX "LiteLLM_VerificationToken_team_id_idx" ON "LiteLLM_VerificationToken"("team_id");
-- CreateIndex
CREATE INDEX "LiteLLM_VerificationToken_budget_reset_at_expires_idx" ON "LiteLLM_VerificationToken"("budget_reset_at", "expires");

13
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -113,6 +113,7 @@ model LiteLLM_TeamTable {
members_with_roles Json @default("{}")
metadata Json @default("{}")
max_budget Float?
soft_budget Float?
spend Float @default(0.0)
models String[]
max_parallel_requests Int?
@ -147,6 +148,7 @@ model LiteLLM_DeletedTeamTable {
members_with_roles Json @default("{}")
metadata Json @default("{}")
max_budget Float?
soft_budget Float?
spend Float @default(0.0)
models String[]
max_parallel_requests Int?
@ -262,6 +264,7 @@ model LiteLLM_MCPServerTable {
token_url String?
registration_url String?
allow_all_keys Boolean @default(false)
available_on_public_internet Boolean @default(false)
}
// Generate Tokens for Proxy
@ -307,6 +310,16 @@ model LiteLLM_VerificationToken {
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
// SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub"
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."user_id" = $1 OFFSET $2
@@index([user_id, team_id])
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."team_id" = $1 OFFSET $2
@@index([team_id])
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE (("public"."LiteLLM_VerificationToken"."expires" IS NULL OR "public"."LiteLLM_VerificationToken"."expires" > $1) AND "public"."LiteLLM_VerificationToken"."budget_reset_at" < $2) OFFSET $3
@@index([budget_reset_at, expires])
}
// Audit table for deleted keys - preserves spend and key information for historical tracking

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.30"
version = "0.4.33"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.30"
version = "0.4.33"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

14
litellm/a2a_protocol/__init__.py
View file

@ -39,6 +39,12 @@ Example usage (class-based):
"""
from litellm.a2a_protocol.client import A2AClient
from litellm.a2a_protocol.exceptions import (
A2AAgentCardError,
A2AConnectionError,
A2AError,
A2ALocalhostURLError,
)
from litellm.a2a_protocol.main import (
aget_agent_card,
asend_message,
@ -49,11 +55,19 @@ from litellm.a2a_protocol.main import (
from litellm.types.agents import LiteLLMSendMessageResponse
__all__ = [
# Client
"A2AClient",
# Functions
"asend_message",
"send_message",
"asend_message_streaming",
"aget_agent_card",
"create_a2a_client",
# Response types
"LiteLLMSendMessageResponse",
# Exceptions
"A2AError",
"A2AConnectionError",
"A2AAgentCardError",
"A2ALocalhostURLError",
]

67
litellm/a2a_protocol/card_resolver.py
View file

@ -7,6 +7,7 @@ Extends the A2A SDK's card resolver to support multiple well-known paths.
from typing import TYPE_CHECKING, Any, Dict, Optional
from litellm._logging import verbose_logger
from litellm.constants import LOCALHOST_URL_PATTERNS
if TYPE_CHECKING:
from a2a.types import AgentCard
@ -26,15 +27,61 @@ except ImportError:
pass
def is_localhost_or_internal_url(url: Optional[str]) -> bool:
"""
Check if a URL is a localhost or internal URL.
This detects common development URLs that are accidentally left in
agent cards when deploying to production.
Args:
url: The URL to check
Returns:
True if the URL is localhost/internal
"""
if not url:
return False
url_lower = url.lower()
return any(pattern in url_lower for pattern in LOCALHOST_URL_PATTERNS)
def fix_agent_card_url(agent_card: "AgentCard", base_url: str) -> "AgentCard":
"""
Fix the agent card URL if it contains a localhost/internal address.
Many A2A agents are deployed with agent cards that contain internal URLs
like "http://0.0.0.0:8001/" or "http://localhost:8000/". This function
replaces such URLs with the provided base_url.
Args:
agent_card: The agent card to fix
base_url: The base URL to use as replacement
Returns:
The agent card with the URL fixed if necessary
"""
card_url = getattr(agent_card, "url", None)
if card_url and is_localhost_or_internal_url(card_url):
# Normalize base_url to ensure it ends with /
fixed_url = base_url.rstrip("/") + "/"
agent_card.url = fixed_url
return agent_card
class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
"""
Custom A2A card resolver that supports multiple well-known paths.
Extends the base A2ACardResolver to try both:
- /.well-known/agent-card.json (standard)
- /.well-known/agent.json (previous/alternative)
"""
async def get_agent_card(
self,
relative_card_path: Optional[str] = None,
@ -42,17 +89,17 @@ class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
) -> "AgentCard":
"""
Fetch the agent card, trying multiple well-known paths.
First tries the standard path, then falls back to the previous path.
Args:
relative_card_path: Optional path to the agent card endpoint.
If None, tries both well-known paths.
http_kwargs: Optional dictionary of keyword arguments to pass to httpx.get
Returns:
AgentCard from the A2A agent
Raises:
A2AClientHTTPError or A2AClientJSONError if both paths fail
"""
@ -62,13 +109,13 @@ class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
relative_card_path=relative_card_path,
http_kwargs=http_kwargs,
)
# Try both well-known paths
paths = [
AGENT_CARD_WELL_KNOWN_PATH,
PREV_AGENT_CARD_WELL_KNOWN_PATH,
]
last_error = None
for path in paths:
try:
@ -85,11 +132,11 @@ class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
)
last_error = e
continue
# If we get here, all paths failed - re-raise the last error
if last_error is not None:
raise last_error
# This shouldn't happen, but just in case
raise Exception(
f"Failed to fetch agent card from {self.base_url}. "

203
litellm/a2a_protocol/exception_mapping_utils.py Normal file
View file

@ -0,0 +1,203 @@
"""
A2A Protocol Exception Mapping Utils.
Maps A2A SDK exceptions to LiteLLM A2A exception types.
"""
from typing import TYPE_CHECKING, Any, Optional
from litellm._logging import verbose_logger
from litellm.a2a_protocol.card_resolver import (
fix_agent_card_url,
is_localhost_or_internal_url,
)
from litellm.a2a_protocol.exceptions import (
A2AAgentCardError,
A2AConnectionError,
A2AError,
A2ALocalhostURLError,
)
from litellm.constants import CONNECTION_ERROR_PATTERNS
if TYPE_CHECKING:
from a2a.client import A2AClient as A2AClientType
# Runtime import
A2A_SDK_AVAILABLE = False
try:
from a2a.client import A2AClient as _A2AClient # type: ignore[no-redef]
A2A_SDK_AVAILABLE = True
except ImportError:
_A2AClient = None # type: ignore[assignment, misc]
class A2AExceptionCheckers:
"""
Helper class for checking various A2A error conditions.
"""
@staticmethod
def is_connection_error(error_str: str) -> bool:
"""
Check if an error string indicates a connection error.
Args:
error_str: The error string to check
Returns:
True if the error indicates a connection issue
"""
if not isinstance(error_str, str):
return False
error_str_lower = error_str.lower()
return any(pattern in error_str_lower for pattern in CONNECTION_ERROR_PATTERNS)
@staticmethod
def is_localhost_url(url: Optional[str]) -> bool:
"""
Check if a URL is a localhost/internal URL.
Args:
url: The URL to check
Returns:
True if the URL is localhost/internal
"""
return is_localhost_or_internal_url(url)
@staticmethod
def is_agent_card_error(error_str: str) -> bool:
"""
Check if an error string indicates an agent card error.
Args:
error_str: The error string to check
Returns:
True if the error is related to agent card fetching/parsing
"""
if not isinstance(error_str, str):
return False
error_str_lower = error_str.lower()
agent_card_patterns = [
"agent card",
"agent-card",
".well-known",
"card not found",
"invalid agent",
]
return any(pattern in error_str_lower for pattern in agent_card_patterns)
def map_a2a_exception(
original_exception: Exception,
card_url: Optional[str] = None,
api_base: Optional[str] = None,
model: Optional[str] = None,
) -> Exception:
"""
Map an A2A SDK exception to a LiteLLM A2A exception type.
Args:
original_exception: The original exception from the A2A SDK
card_url: The URL from the agent card (if available)
api_base: The original API base URL
model: The model/agent name
Returns:
A mapped LiteLLM A2A exception
Raises:
A2ALocalhostURLError: If the error is a connection error to a localhost URL
A2AConnectionError: If the error is a general connection error
A2AAgentCardError: If the error is related to agent card issues
A2AError: For other A2A-related errors
"""
error_str = str(original_exception)
# Check for localhost URL connection error (special case - retryable)
if (
card_url
and api_base
and A2AExceptionCheckers.is_localhost_url(card_url)
and A2AExceptionCheckers.is_connection_error(error_str)
):
raise A2ALocalhostURLError(
localhost_url=card_url,
base_url=api_base,
original_error=original_exception,
model=model,
)
# Check for agent card errors
if A2AExceptionCheckers.is_agent_card_error(error_str):
raise A2AAgentCardError(
message=error_str,
url=api_base,
model=model,
)
# Check for general connection errors
if A2AExceptionCheckers.is_connection_error(error_str):
raise A2AConnectionError(
message=error_str,
url=card_url or api_base,
model=model,
)
# Default: wrap in generic A2AError
raise A2AError(
message=error_str,
model=model,
)
def handle_a2a_localhost_retry(
error: A2ALocalhostURLError,
agent_card: Any,
a2a_client: "A2AClientType",
is_streaming: bool = False,
) -> "A2AClientType":
"""
Handle A2ALocalhostURLError by fixing the URL and creating a new client.
This is called when we catch an A2ALocalhostURLError and want to retry
with the corrected URL.
Args:
error: The localhost URL error
agent_card: The agent card object to fix
a2a_client: The current A2A client
is_streaming: Whether this is a streaming request (for logging)
Returns:
A new A2A client with the fixed URL
Raises:
ImportError: If the A2A SDK is not installed
"""
if not A2A_SDK_AVAILABLE or _A2AClient is None:
raise ImportError(
"A2A SDK is required for localhost retry handling. "
"Install it with: pip install a2a"
)
request_type = "streaming " if is_streaming else ""
verbose_logger.warning(
f"A2A {request_type}request to '{error.localhost_url}' failed: {error.original_error}. "
f"Agent card contains localhost/internal URL. "
f"Retrying with base_url '{error.base_url}'."
)
# Fix the agent card URL
fix_agent_card_url(agent_card, error.base_url)
# Create a new client with the fixed agent card (transport caches URL)
return _A2AClient(
httpx_client=a2a_client._transport.httpx_client, # type: ignore[union-attr]
agent_card=agent_card,
)

150
litellm/a2a_protocol/exceptions.py Normal file
View file

@ -0,0 +1,150 @@
"""
A2A Protocol Exceptions.
Custom exception types for A2A protocol operations, following LiteLLM's exception pattern.
"""
from typing import Optional
import httpx
class A2AError(Exception):
"""
Base exception for A2A protocol errors.
Follows the same pattern as LiteLLM's main exceptions.
"""
def __init__(
self,
message: str,
status_code: int = 500,
llm_provider: str = "a2a_agent",
model: Optional[str] = None,
response: Optional[httpx.Response] = None,
litellm_debug_info: Optional[str] = None,
max_retries: Optional[int] = None,
num_retries: Optional[int] = None,
):
self.status_code = status_code
self.message = f"litellm.A2AError: {message}"
self.llm_provider = llm_provider
self.model = model
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
self.response = response or httpx.Response(
status_code=self.status_code,
request=httpx.Request(method="POST", url="https://litellm.ai"),
)
super().__init__(self.message)
def __str__(self) -> str:
_message = self.message
if self.num_retries:
_message += f" LiteLLM Retried: {self.num_retries} times"
if self.max_retries:
_message += f", LiteLLM Max Retries: {self.max_retries}"
return _message
def __repr__(self) -> str:
return self.__str__()
class A2AConnectionError(A2AError):
"""
Raised when connection to an A2A agent fails.
This typically occurs when:
- The agent is unreachable
- The agent card contains a localhost/internal URL
- Network issues prevent connection
"""
def __init__(
self,
message: str,
url: Optional[str] = None,
model: Optional[str] = None,
response: Optional[httpx.Response] = None,
litellm_debug_info: Optional[str] = None,
max_retries: Optional[int] = None,
num_retries: Optional[int] = None,
):
self.url = url
super().__init__(
message=message,
status_code=503,
llm_provider="a2a_agent",
model=model,
response=response,
litellm_debug_info=litellm_debug_info,
max_retries=max_retries,
num_retries=num_retries,
)
class A2AAgentCardError(A2AError):
"""
Raised when there's an issue with the agent card.
This includes:
- Failed to fetch agent card
- Invalid agent card format
- Missing required fields
"""
def __init__(
self,
message: str,
url: Optional[str] = None,
model: Optional[str] = None,
response: Optional[httpx.Response] = None,
litellm_debug_info: Optional[str] = None,
):
self.url = url
super().__init__(
message=message,
status_code=404,
llm_provider="a2a_agent",
model=model,
response=response,
litellm_debug_info=litellm_debug_info,
)
class A2ALocalhostURLError(A2AConnectionError):
"""
Raised when an agent card contains a localhost/internal URL.
Many A2A agents are deployed with agent cards that contain internal URLs
like "http://0.0.0.0:8001/" or "http://localhost:8000/". This error
indicates that the URL needs to be corrected and the request should be retried.
Attributes:
localhost_url: The localhost/internal URL found in the agent card
base_url: The public base URL that should be used instead
original_error: The original connection error that was raised
"""
def __init__(
self,
localhost_url: str,
base_url: str,
original_error: Optional[Exception] = None,
model: Optional[str] = None,
):
self.localhost_url = localhost_url
self.base_url = base_url
self.original_error = original_error
message = (
f"Agent card contains localhost/internal URL '{localhost_url}'. "
f"Retrying with base URL '{base_url}'."
)
super().__init__(
message=message,
url=localhost_url,
model=model,
)

184
litellm/a2a_protocol/main.py
View file

@ -44,6 +44,11 @@ except ImportError:
# Import our custom card resolver that supports multiple well-known paths
from litellm.a2a_protocol.card_resolver import LiteLLMA2ACardResolver
from litellm.a2a_protocol.exception_mapping_utils import (
handle_a2a_localhost_retry,
map_a2a_exception,
)
from litellm.a2a_protocol.exceptions import A2ALocalhostURLError
# Use our custom resolver instead of the default A2A SDK resolver
A2ACardResolver = LiteLLMA2ACardResolver
@ -244,10 +249,50 @@ async def asend_message(
verbose_logger.info(f"A2A send_message request_id={request.id}, agent={agent_name}")
a2a_response = await a2a_client.send_message(request)
# Get agent card URL for localhost retry logic
agent_card = getattr(a2a_client, "_litellm_agent_card", None) or getattr(
a2a_client, "agent_card", None
)
card_url = getattr(agent_card, "url", None) if agent_card else None
# Retry loop: if connection fails due to localhost URL in agent card, retry with fixed URL
a2a_response = None
for _ in range(2): # max 2 attempts: original + 1 retry
try:
a2a_response = await a2a_client.send_message(request)
break # success, exit retry loop
except A2ALocalhostURLError as e:
# Localhost URL error - fix and retry
a2a_client = handle_a2a_localhost_retry(
error=e,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=False,
)
card_url = agent_card.url if agent_card else None
except Exception as e:
# Map exception - will raise A2ALocalhostURLError if applicable
try:
map_a2a_exception(e, card_url, api_base, model=agent_name)
except A2ALocalhostURLError as localhost_err:
# Localhost URL error - fix and retry
a2a_client = handle_a2a_localhost_retry(
error=localhost_err,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=False,
)
card_url = agent_card.url if agent_card else None
continue
except Exception:
# Re-raise the mapped exception
raise
verbose_logger.info(f"A2A send_message completed, request_id={request.id}")
# a2a_response is guaranteed to be set if we reach here (loop breaks on success or raises)
assert a2a_response is not None
# Wrap in LiteLLM response type for _hidden_params support
response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response)
@ -307,6 +352,48 @@ def send_message(
)
def _build_streaming_logging_obj(
request: "SendStreamingMessageRequest",
agent_name: str,
agent_id: Optional[str],
litellm_params: Optional[Dict[str, Any]],
metadata: Optional[Dict[str, Any]],
proxy_server_request: Optional[Dict[str, Any]],
) -> Logging:
"""Build logging object for streaming A2A requests."""
start_time = datetime.datetime.now()
model = f"a2a_agent/{agent_name}"
logging_obj = Logging(
model=model,
messages=[{"role": "user", "content": "streaming-request"}],
stream=False,
call_type="asend_message_streaming",
start_time=start_time,
litellm_call_id=str(request.id),
function_id=str(request.id),
)
logging_obj.model = model
logging_obj.custom_llm_provider = "a2a_agent"
logging_obj.model_call_details["model"] = model
logging_obj.model_call_details["custom_llm_provider"] = "a2a_agent"
if agent_id:
logging_obj.model_call_details["agent_id"] = agent_id
_litellm_params = litellm_params.copy() if litellm_params else {}
if metadata:
_litellm_params["metadata"] = metadata
if proxy_server_request:
_litellm_params["proxy_server_request"] = proxy_server_request
logging_obj.litellm_params = _litellm_params
logging_obj.optional_params = _litellm_params
logging_obj.model_call_details["litellm_params"] = _litellm_params
logging_obj.model_call_details["metadata"] = metadata or {}
return logging_obj
async def asend_message_streaming(
a2a_client: Optional["A2AClientType"] = None,
request: Optional["SendStreamingMessageRequest"] = None,
@ -403,55 +490,72 @@ async def asend_message_streaming(
verbose_logger.info(f"A2A send_message_streaming request_id={request.id}")
# Track for logging
start_time = datetime.datetime.now()
stream = a2a_client.send_message_streaming(request)
# Build logging object for streaming completion callbacks
agent_card = getattr(a2a_client, "_litellm_agent_card", None) or getattr(
a2a_client, "agent_card", None
)
card_url = getattr(agent_card, "url", None) if agent_card else None
agent_name = getattr(agent_card, "name", "unknown") if agent_card else "unknown"
model = f"a2a_agent/{agent_name}"
logging_obj = Logging(
model=model,
messages=[{"role": "user", "content": "streaming-request"}],
stream=False, # complete response logging after stream ends
call_type="asend_message_streaming",
start_time=start_time,
litellm_call_id=str(request.id),
function_id=str(request.id),
)
logging_obj.model = model
logging_obj.custom_llm_provider = "a2a_agent"
logging_obj.model_call_details["model"] = model
logging_obj.model_call_details["custom_llm_provider"] = "a2a_agent"
if agent_id:
logging_obj.model_call_details["agent_id"] = agent_id
# Propagate litellm_params for spend logging (includes cost_per_query, etc.)
_litellm_params = litellm_params.copy() if litellm_params else {}
# Merge metadata into litellm_params.metadata (required for proxy cost tracking)
if metadata:
_litellm_params["metadata"] = metadata
if proxy_server_request:
_litellm_params["proxy_server_request"] = proxy_server_request
logging_obj.litellm_params = _litellm_params
logging_obj.optional_params = _litellm_params # used by cost calc
logging_obj.model_call_details["litellm_params"] = _litellm_params
logging_obj.model_call_details["metadata"] = metadata or {}
iterator = A2AStreamingIterator(
stream=stream,
logging_obj = _build_streaming_logging_obj(
request=request,
logging_obj=logging_obj,
agent_name=agent_name,
agent_id=agent_id,
litellm_params=litellm_params,
metadata=metadata,
proxy_server_request=proxy_server_request,
)
async for chunk in iterator:
yield chunk
# Retry loop: if connection fails due to localhost URL in agent card, retry with fixed URL
# Connection errors in streaming typically occur on first chunk iteration
first_chunk = True
for attempt in range(2): # max 2 attempts: original + 1 retry
stream = a2a_client.send_message_streaming(request)
iterator = A2AStreamingIterator(
stream=stream,
request=request,
logging_obj=logging_obj,
agent_name=agent_name,
)
try:
first_chunk = True
async for chunk in iterator:
if first_chunk:
first_chunk = False # connection succeeded
yield chunk
return # stream completed successfully
except A2ALocalhostURLError as e:
# Only retry on first chunk, not mid-stream
if first_chunk and attempt == 0:
a2a_client = handle_a2a_localhost_retry(
error=e,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=True,
)
card_url = agent_card.url if agent_card else None
else:
raise
except Exception as e:
# Only map exception on first chunk
if first_chunk and attempt == 0:
try:
map_a2a_exception(e, card_url, api_base, model=agent_name)
except A2ALocalhostURLError as localhost_err:
# Localhost URL error - fix and retry
a2a_client = handle_a2a_localhost_retry(
error=localhost_err,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=True,
)
card_url = agent_card.url if agent_card else None
continue
except Exception:
# Re-raise the mapped exception
raise
raise
async def create_a2a_client(

30
litellm/anthropic_beta_headers_config.json Normal file
View file

@ -0,0 +1,30 @@
{
"description": "Unsupported Anthropic beta headers for each provider. Headers listed here will be dropped. Headers not listed are passed through as-is.",
"anthropic": [],
"azure_ai": [],
"bedrock_converse": [
"prompt-caching-scope-2026-01-05",
"bash_20250124",
"bash_20241022",
"text_editor_20250124",
"text_editor_20241022",
"compact-2026-01-12",
"advanced-tool-use-2025-11-20",
"web-fetch-2025-09-10",
"code-execution-2025-08-25",
"skills-2025-10-02",
"files-api-2025-04-14"
],
"bedrock": [
"advanced-tool-use-2025-11-20",
"prompt-caching-scope-2026-01-05",
"structured-outputs-2025-11-13",
"web-fetch-2025-09-10",
"code-execution-2025-08-25",
"skills-2025-10-02",
"files-api-2025-04-14"
],
"vertex_ai": [
"prompt-caching-scope-2026-01-05"
]
}

221
litellm/anthropic_beta_headers_manager.py Normal file
View file

@ -0,0 +1,221 @@
"""
Centralized manager for Anthropic beta headers across different providers.
This module provides utilities to:
1. Load beta header configuration from JSON (lists unsupported headers per provider)
2. Filter out unsupported beta headers
3. Handle provider-specific header name mappings (e.g., advanced-tool-use -> tool-search-tool)
Design:
- JSON config lists UNSUPPORTED headers for each provider
- Headers not in the unsupported list are passed through
- Header mappings allow renaming headers for specific providers
"""
import json
import os
from typing import Dict, List, Optional, Set
from litellm.litellm_core_utils.litellm_logging import verbose_logger
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: Optional[Dict] = None
def _load_beta_headers_config() -> Dict:
"""
Load the beta headers configuration from JSON file.
Uses caching to avoid repeated file reads.
Returns:
Dict containing the beta headers configuration
"""
global _BETA_HEADERS_CONFIG
if _BETA_HEADERS_CONFIG is not None:
return _BETA_HEADERS_CONFIG
config_path = os.path.join(
os.path.dirname(__file__),
"anthropic_beta_headers_config.json"
)
try:
with open(config_path, "r") as f:
_BETA_HEADERS_CONFIG = json.load(f)
verbose_logger.debug(f"Loaded beta headers config from {config_path}")
return _BETA_HEADERS_CONFIG
except Exception as e:
verbose_logger.error(f"Failed to load beta headers config: {e}")
# Return empty config as fallback
return {
"anthropic": [],
"azure_ai": [],
"bedrock": [],
"bedrock_converse": [],
"vertex_ai": []
}
def get_provider_name(provider: str) -> str:
"""
Resolve provider aliases to canonical provider names.
Args:
provider: Provider name (may be an alias)
Returns:
Canonical provider name
"""
config = _load_beta_headers_config()
aliases = config.get("provider_aliases", {})
return aliases.get(provider, provider)
def filter_and_transform_beta_headers(
beta_headers: List[str],
provider: str,
) -> List[str]:
"""
Filter beta headers based on provider's unsupported list.
This function:
1. Removes headers that are in the provider's unsupported list
2. Passes through all other headers as-is
Note: Header transformations/mappings (e.g., advanced-tool-use -> tool-search-tool)
are handled in each provider's transformation code, not here.
Args:
beta_headers: List of Anthropic beta header values
provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
Returns:
List of filtered beta headers for the provider
"""
if not beta_headers:
return []
config = _load_beta_headers_config()
provider = get_provider_name(provider)
# Get unsupported headers for this provider
unsupported_headers = set(config.get(provider, []))
filtered_headers: Set[str] = set()
for header in beta_headers:
header = header.strip()
# Skip if header is unsupported
if header in unsupported_headers:
verbose_logger.debug(
f"Dropping unsupported beta header '{header}' for provider '{provider}'"
)
continue
# Pass through as-is
filtered_headers.add(header)
return sorted(list(filtered_headers))
def is_beta_header_supported(
beta_header: str,
provider: str,
) -> bool:
"""
Check if a specific beta header is supported by a provider.
Args:
beta_header: The Anthropic beta header value
provider: Provider name
Returns:
True if the header is supported (not in unsupported list), False otherwise
"""
config = _load_beta_headers_config()
provider = get_provider_name(provider)
unsupported_headers = set(config.get(provider, []))
return beta_header not in unsupported_headers
def get_provider_beta_header(
anthropic_beta_header: str,
provider: str,
) -> Optional[str]:
"""
Check if a beta header is supported by a provider.
Note: This does NOT handle header transformations/mappings.
Those are handled in each provider's transformation code.
Args:
anthropic_beta_header: The Anthropic beta header value
provider: Provider name
Returns:
The original header if supported, or None if unsupported
"""
config = _load_beta_headers_config()
provider = get_provider_name(provider)
# Check if unsupported
unsupported_headers = set(config.get(provider, []))
if anthropic_beta_header in unsupported_headers:
return None
return anthropic_beta_header
def update_headers_with_filtered_beta(
headers: dict,
provider: str,
) -> dict:
"""
Update headers dict by filtering and transforming anthropic-beta header values.
Modifies the headers dict in place and returns it.
Args:
headers: Request headers dict (will be modified in place)
provider: Provider name
Returns:
Updated headers dict
"""
existing_beta = headers.get("anthropic-beta")
if not existing_beta:
return headers
# Parse existing beta headers
beta_values = [b.strip() for b in existing_beta.split(",") if b.strip()]
# Filter and transform based on provider
filtered_beta_values = filter_and_transform_beta_headers(
beta_headers=beta_values,
provider=provider,
)
# Update or remove the header
if filtered_beta_values:
headers["anthropic-beta"] = ",".join(filtered_beta_values)
else:
# Remove the header if no values remain
headers.pop("anthropic-beta", None)
return headers
def get_unsupported_headers(provider: str) -> List[str]:
"""
Get all beta headers that are unsupported by a provider.
Args:
provider: Provider name
Returns:
List of unsupported Anthropic beta header names
"""
config = _load_beta_headers_config()
provider = get_provider_name(provider)
return config.get(provider, [])

2
litellm/caching/redis_cache.py
View file

@ -1123,7 +1123,7 @@ class RedisCache(BaseCache):
redis_client = redis_async.Redis(**self.redis_kwargs)
# Test the connection
ping_result = await redis_client.ping()
ping_result = await redis_client.ping() # type: ignore[misc]
# Close the connection
await redis_client.aclose() # type: ignore[attr-defined]

2
litellm/caching/redis_cluster_cache.py
View file

@ -83,7 +83,7 @@ class RedisClusterCache(RedisCache):
)
# Test the connection
ping_result = await redis_client.ping() # type: ignore[attr-defined]
ping_result = await redis_client.ping() # type: ignore[attr-defined, misc]
# Close the connection
await redis_client.aclose() # type: ignore[attr-defined]

18
litellm/constants.py
View file

@ -306,6 +306,22 @@ DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2
#### Networking settings ####
request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds
DEFAULT_A2A_AGENT_TIMEOUT: float = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)) # 10 minutes
# Patterns that indicate a localhost/internal URL in A2A agent cards that should be
# replaced with the original base_url. This is a common misconfiguration where
# developers deploy agents with development URLs in their agent cards.
LOCALHOST_URL_PATTERNS: List[str] = [
"localhost",
"127.0.0.1",
"0.0.0.0",
"[::1]", # IPv6 localhost
]
# Patterns in error messages that indicate a connection failure
CONNECTION_ERROR_PATTERNS: List[str] = [
"connect",
"connection",
"network",
"refused",
]
STREAM_SSE_DONE_STRING: str = "[DONE]"
STREAM_SSE_DATA_PREFIX: str = "data: "
### SPEND TRACKING ###
@ -970,6 +986,8 @@ BEDROCK_CONVERSE_MODELS = [
"openai.gpt-oss-120b-1:0",
"anthropic.claude-haiku-4-5-20251001-v1:0",
"anthropic.claude-sonnet-4-5-20250929-v1:0",
"anthropic.claude-opus-4-6-v1:0",
"anthropic.claude-opus-4-6-v1",
"anthropic.claude-opus-4-1-20250805-v1:0",
"anthropic.claude-opus-4-20250514-v1:0",
"anthropic.claude-sonnet-4-20250514-v1:0",

113
litellm/cost_calculator.py
View file

@ -1,5 +1,6 @@
# What is this?
## File for 'response_cost' calculation in Logging
import logging
import time
from functools import lru_cache
from typing import TYPE_CHECKING, Any, List, Literal, Optional, Tuple, Union, cast
@ -774,10 +775,11 @@ def _apply_cost_discount(
discount_amount = original_cost * discount_percent
final_cost = original_cost - discount_amount
verbose_logger.debug(
f"Applied {discount_percent*100}% discount to {custom_llm_provider}: "
f"${original_cost:.6f} -> ${final_cost:.6f} (saved ${discount_amount:.6f})"
)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
f"Applied {discount_percent*100}% discount to {custom_llm_provider}: "
f"${original_cost:.6f} -> ${final_cost:.6f} (saved ${discount_amount:.6f})"
)
return final_cost, discount_percent, discount_amount
@ -807,17 +809,20 @@ def _apply_cost_margin(
margin_config = None
if custom_llm_provider and custom_llm_provider in litellm.cost_margin_config:
margin_config = litellm.cost_margin_config[custom_llm_provider]
verbose_logger.debug(
f"Found provider-specific margin config for {custom_llm_provider}: {margin_config}"
)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
f"Found provider-specific margin config for {custom_llm_provider}: {margin_config}"
)
elif "global" in litellm.cost_margin_config:
margin_config = litellm.cost_margin_config["global"]
verbose_logger.debug(f"Using global margin config: {margin_config}")
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(f"Using global margin config: {margin_config}")
else:
verbose_logger.debug(
f"No margin config found. Provider: {custom_llm_provider}, "
f"Available configs: {list(litellm.cost_margin_config.keys())}"
)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
f"No margin config found. Provider: {custom_llm_provider}, "
f"Available configs: {list(litellm.cost_margin_config.keys())}"
)
if margin_config is not None:
# Handle different margin config formats
@ -836,11 +841,12 @@ def _apply_cost_margin(
final_cost = original_cost + margin_total_amount
verbose_logger.debug(
f"Applied margin to {custom_llm_provider or 'global'}: "
f"${original_cost:.6f} -> ${final_cost:.6f} "
f"(margin: {margin_percent*100 if margin_percent > 0 else 0}% + ${margin_fixed_amount:.6f} = ${margin_total_amount:.6f})"
)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
f"Applied margin to {custom_llm_provider or 'global'}: "
f"${original_cost:.6f} -> ${final_cost:.6f} "
f"(margin: {margin_percent*100 if margin_percent > 0 else 0}% + ${margin_fixed_amount:.6f} = ${margin_total_amount:.6f})"
)
return final_cost, margin_percent, margin_fixed_amount, margin_total_amount
@ -1021,9 +1027,10 @@ def completion_cost( # noqa: PLR0915
for idx, model in enumerate(potential_model_names):
try:
verbose_logger.debug(
f"selected model name for cost calculation: {model}"
)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
f"selected model name for cost calculation: {model}"
)
if completion_response is not None and (
isinstance(completion_response, BaseModel)
@ -1411,37 +1418,47 @@ def completion_cost( # noqa: PLR0915
# Apply discount from module-level config if configured
original_cost = _final_cost
_final_cost, discount_percent, discount_amount = _apply_cost_discount(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
if litellm.cost_discount_config:
_final_cost, discount_percent, discount_amount = _apply_cost_discount(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
else:
discount_percent = 0.0
discount_amount = 0.0
# Apply margin from module-level config if configured
(
_final_cost,
margin_percent,
margin_fixed_amount,
margin_total_amount,
) = _apply_cost_margin(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
if litellm.cost_margin_config:
(
_final_cost,
margin_percent,
margin_fixed_amount,
margin_total_amount,
) = _apply_cost_margin(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
else:
margin_percent = 0.0
margin_fixed_amount = 0.0
margin_total_amount = 0.0
# Store cost breakdown in logging object if available
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar=completion_tokens_cost_usd_dollar,
cost_for_built_in_tools_cost_usd_dollar=cost_for_built_in_tools,
total_cost_usd_dollar=_final_cost,
additional_costs=additional_costs,
original_cost=original_cost,
discount_percent=discount_percent,
discount_amount=discount_amount,
margin_percent=margin_percent,
margin_fixed_amount=margin_fixed_amount,
margin_total_amount=margin_total_amount,
)
if litellm_logging_obj is not None:
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar=completion_tokens_cost_usd_dollar,
cost_for_built_in_tools_cost_usd_dollar=cost_for_built_in_tools,
total_cost_usd_dollar=_final_cost,
original_cost=original_cost,
additional_costs=additional_costs,
discount_percent=discount_percent,
discount_amount=discount_amount,
margin_percent=margin_percent,
margin_fixed_amount=margin_fixed_amount,
margin_total_amount=margin_total_amount,
)
return _final_cost
except Exception as e:
@ -2116,3 +2133,5 @@ def handle_realtime_stream_cost_calculation(
total_cost = input_cost_per_token + output_cost_per_token
return total_cost

2
litellm/experimental_mcp_client/client.py
View file

@ -209,6 +209,8 @@ class MCPClient:
headers["X-API-Key"] = self._mcp_auth_value
elif self.auth_type == MCPAuth.authorization:
headers["Authorization"] = self._mcp_auth_value
elif self.auth_type == MCPAuth.oauth2:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
elif isinstance(self._mcp_auth_value, dict):
headers.update(self._mcp_auth_value)

1
litellm/integrations/custom_guardrail.py
View file

@ -268,6 +268,7 @@ class CustomGuardrail(CustomLogger):
"""
Returns the guardrail(s) to be run from the metadata or root
"""
if "guardrails" in data:
return data["guardrails"]
metadata = data.get("litellm_metadata") or data.get("metadata", {})

24
litellm/integrations/email_templates/templates.py
View file

@ -85,6 +85,30 @@ SOFT_BUDGET_ALERT_EMAIL_TEMPLATE = """
The LiteLLM team <br />
"""
TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE = """
<img src="{email_logo_url}" alt="LiteLLM Logo" width="150" height="50" />
<p> Hi {team_alias} team member, <br/>
Your LiteLLM team has crossed its <b>soft budget limit of {soft_budget}</b>. <br /> <br />
<b>Current Spend:</b> {spend} <br />
<b>Soft Budget:</b> {soft_budget} <br />
{max_budget_info}
<p style="color: #dc2626; font-weight: 500;">
⚠️ Note: Your API requests will continue to work, but you should monitor your usage closely.
If you reach your maximum budget, requests will be rejected.
</p>
You can view your usage and manage your budget in the <a href="{base_url}">LiteLLM Dashboard</a>. <br /> <br />
If you have any questions, please send an email to {email_support_contact} <br /> <br />
Best, <br />
The LiteLLM team <br />
"""
MAX_BUDGET_ALERT_EMAIL_TEMPLATE = """
<img src="{email_logo_url}" alt="LiteLLM Logo" width="150" height="50" />

79
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -8,9 +8,8 @@ from litellm.integrations.arize import _utils
from litellm.integrations.langfuse.langfuse_otel_attributes import (
LangfuseLLMObsOTELAttributes,
)
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.types.integrations.langfuse_otel import (
LangfuseOtelConfig,
LangfuseSpanAttributes,
)
from litellm.types.utils import StandardCallbackDynamicParams
@ -18,17 +17,8 @@ from litellm.types.utils import StandardCallbackDynamicParams
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.integrations.opentelemetry import (
OpenTelemetryConfig as _OpenTelemetryConfig,
)
from litellm.types.integrations.arize import Protocol as _Protocol
Protocol = _Protocol
OpenTelemetryConfig = _OpenTelemetryConfig
Span = Union[_Span, Any]
else:
Protocol = Any
OpenTelemetryConfig = Any
Span = Any
@ -37,8 +27,12 @@ LANGFUSE_CLOUD_US_ENDPOINT = "https://us.cloud.langfuse.com/api/public/otel"
class LangfuseOtelLogger(OpenTelemetry):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def __init__(self, config=None, *args, **kwargs):
# Prevent LangfuseOtelLogger from modifying global environment variables by constructing config manually
# and passing it to the parent OpenTelemetry class
if config is None:
config = self._create_open_telemetry_config_from_langfuse_env()
super().__init__(config=config, *args, **kwargs)
@staticmethod
def set_langfuse_otel_attributes(span: Span, kwargs, response_obj):
@ -114,6 +108,10 @@ class LangfuseOtelLogger(OpenTelemetry):
for key, enum_attr in mapping.items():
if key in metadata and metadata[key] is not None:
value = metadata[key]
if key == "trace_id" and isinstance(value, str):
# trace_id must be 32 hex char no dashes for langfuse : Litellm sends uuid with dashes (might be breaking at some point)
value = value.replace("-", "")
if isinstance(value, (list, dict)):
try:
value = json.dumps(value)
@ -265,8 +263,47 @@ class LangfuseOtelLogger(OpenTelemetry):
"""
return os.environ.get("LANGFUSE_OTEL_HOST") or os.environ.get("LANGFUSE_HOST")
def _create_open_telemetry_config_from_langfuse_env(self) -> OpenTelemetryConfig:
"""
Creates OpenTelemetryConfig from Langfuse environment variables.
Does NOT modify global environment variables.
"""
from litellm.integrations.opentelemetry import OpenTelemetryConfig
public_key = os.environ.get("LANGFUSE_PUBLIC_KEY", None)
secret_key = os.environ.get("LANGFUSE_SECRET_KEY", None)
if not public_key or not secret_key:
# If no keys, return default from env (likely logging to console or something else)
return OpenTelemetryConfig.from_env()
# Determine endpoint - default to US cloud
langfuse_host = LangfuseOtelLogger._get_langfuse_otel_host()
if langfuse_host:
# If LANGFUSE_HOST is provided, construct OTEL endpoint from it
if not langfuse_host.startswith("http"):
langfuse_host = "https://" + langfuse_host
endpoint = f"{langfuse_host.rstrip('/')}/api/public/otel"
verbose_logger.debug(f"Using Langfuse OTEL endpoint from host: {endpoint}")
else:
# Default to US cloud endpoint
endpoint = LANGFUSE_CLOUD_US_ENDPOINT
verbose_logger.debug(f"Using Langfuse US cloud endpoint: {endpoint}")
auth_header = LangfuseOtelLogger._get_langfuse_authorization_header(
public_key=public_key, secret_key=secret_key
)
otlp_auth_headers = f"Authorization={auth_header}"
return OpenTelemetryConfig(
exporter="otlp_http",
endpoint=endpoint,
headers=otlp_auth_headers,
)
@staticmethod
def get_langfuse_otel_config() -> LangfuseOtelConfig:
def get_langfuse_otel_config() -> "OpenTelemetryConfig":
"""
Retrieves the Langfuse OpenTelemetry configuration based on environment variables.
@ -276,7 +313,7 @@ class LangfuseOtelLogger(OpenTelemetry):
LANGFUSE_HOST: Optional. Custom Langfuse host URL. Defaults to US cloud.
Returns:
LangfuseOtelConfig: A Pydantic model containing Langfuse OTEL configuration.
OpenTelemetryConfig: A Pydantic model containing Langfuse OTEL configuration.
Raises:
ValueError: If required keys are missing.
@ -308,12 +345,14 @@ class LangfuseOtelLogger(OpenTelemetry):
)
otlp_auth_headers = f"Authorization={auth_header}"
# Set standard OTEL environment variables
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers
# Prevent modification of global env vars which causes leakage
# os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint
# os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers
return LangfuseOtelConfig(
otlp_auth_headers=otlp_auth_headers, protocol="otlp_http"
return OpenTelemetryConfig(
exporter="otlp_http",
endpoint=endpoint,
headers=otlp_auth_headers,
)
@staticmethod

30
litellm/integrations/opentelemetry.py
View file

@ -599,9 +599,9 @@ class OpenTelemetry(CustomLogger):
def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]:
"""Extract dynamic headers from kwargs if available."""
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params")
)
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params")
if not standard_callback_dynamic_params:
return None
@ -619,7 +619,9 @@ class OpenTelemetry(CustomLogger):
# Prevents thread exhaustion by reusing providers for the same credential sets (e.g. per-team keys)
cache_key = str(sorted(dynamic_headers.items()))
if cache_key in self._tracer_provider_cache:
return self._tracer_provider_cache[cache_key].get_tracer(LITELLM_TRACER_NAME)
return self._tracer_provider_cache[cache_key].get_tracer(
LITELLM_TRACER_NAME
)
# Create a temporary tracer provider with dynamic headers
temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config))
@ -674,7 +676,10 @@ class OpenTelemetry(CustomLogger):
kwargs, response_obj, start_time, end_time, span
)
# Ensure proxy-request parent span is annotated with the actual operation kind
if parent_span is not None and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME:
if (
parent_span is not None
and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
):
self.set_attributes(parent_span, kwargs, response_obj)
else:
# Do not create primary span (keep hierarchy shallow when parent exists)
@ -1003,14 +1008,11 @@ class OpenTelemetry(CustomLogger):
# TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider
try:
from opentelemetry.sdk._logs import (
LogRecord as SdkLogRecord, # type: ignore[attr-defined] # OTEL < 1.39.0
)
from opentelemetry.sdk._logs import LogRecord as SdkLogRecord # type: ignore[attr-defined] # OTEL < 1.39.0
except ImportError:
from opentelemetry.sdk._logs._internal import (
LogRecord as SdkLogRecord, # OTEL >= 1.39.0
)
from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord # type: ignore[attr-defined, no-redef] # OTEL >= 1.39.0
otel_logger = get_logger(LITELLM_LOGGER_NAME)
@ -1618,7 +1620,6 @@ class OpenTelemetry(CustomLogger):
for idx, choice in enumerate(response_obj.get("choices")):
if choice.get("finish_reason"):
message = choice.get("message")
tool_calls = message.get("tool_calls")
if tool_calls:
@ -1631,7 +1632,9 @@ class OpenTelemetry(CustomLogger):
)
except Exception as e:
self.handle_callback_failure(callback_name=self.callback_name or "opentelemetry")
self.handle_callback_failure(
callback_name=self.callback_name or "opentelemetry"
)
verbose_logger.exception(
"OpenTelemetry logging error in set_attributes %s", str(e)
)
@ -1722,6 +1725,7 @@ class OpenTelemetry(CustomLogger):
def set_raw_request_attributes(self, span: Span, kwargs, response_obj):
try:
self.set_attributes(span, kwargs, response_obj)
kwargs.get("optional_params", {})
litellm_params = kwargs.get("litellm_params", {}) or {}
custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown")

7
litellm/integrations/posthog.py
View file

@ -17,6 +17,7 @@ from typing import Any, Dict, Optional, Tuple
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.integrations.posthog_mock_client import (
should_use_posthog_mock,
create_mock_posthog_client,
@ -100,7 +101,7 @@ class PostHogLogger(CustomBatchLogger):
response = self.sync_client.post(
url=capture_url,
json=payload,
content=safe_dumps(payload),
headers=headers,
)
response.raise_for_status()
@ -356,7 +357,7 @@ class PostHogLogger(CustomBatchLogger):
response = await self.async_client.post(
url=capture_url,
json=payload,
content=safe_dumps(payload),
headers=headers,
)
response.raise_for_status()
@ -438,7 +439,7 @@ class PostHogLogger(CustomBatchLogger):
response = self.sync_client.post(
url=capture_url,
json=payload,
content=safe_dumps(payload),
headers=headers,
)
response.raise_for_status()

53
litellm/integrations/prometheus.py
View file

@ -1,6 +1,7 @@
# used for /metrics endpoint on LiteLLM Proxy
#### What this does ####
# On success, log events to Prometheus
import asyncio
import os
import sys
from datetime import datetime, timedelta
@ -1188,28 +1189,34 @@ class PrometheusLogger(CustomLogger):
_user_spend = _metadata.get("user_api_key_user_spend", None)
_user_max_budget = _metadata.get("user_api_key_user_max_budget", None)
await self._set_api_key_budget_metrics_after_api_request(
user_api_key=user_api_key,
user_api_key_alias=user_api_key_alias,
response_cost=response_cost,
key_max_budget=_api_key_max_budget,
key_spend=_api_key_spend,
)
await self._set_team_budget_metrics_after_api_request(
user_api_team=user_api_team,
user_api_team_alias=user_api_team_alias,
team_spend=_team_spend,
team_max_budget=_team_max_budget,
response_cost=response_cost,
)
await self._set_user_budget_metrics_after_api_request(
user_id=user_id,
user_spend=_user_spend,
user_max_budget=_user_max_budget,
response_cost=response_cost,
results = await asyncio.gather(
self._set_api_key_budget_metrics_after_api_request(
user_api_key=user_api_key,
user_api_key_alias=user_api_key_alias,
response_cost=response_cost,
key_max_budget=_api_key_max_budget,
key_spend=_api_key_spend,
),
self._set_team_budget_metrics_after_api_request(
user_api_team=user_api_team,
user_api_team_alias=user_api_team_alias,
team_spend=_team_spend,
team_max_budget=_team_max_budget,
response_cost=response_cost,
),
self._set_user_budget_metrics_after_api_request(
user_id=user_id,
user_spend=_user_spend,
user_max_budget=_user_max_budget,
response_cost=response_cost,
),
return_exceptions=True,
)
for i, r in enumerate(results):
if isinstance(r, Exception):
verbose_logger.debug(
f"[Non-Blocking] Prometheus: Budget metric lookup {['key', 'team', 'user'][i]} failed: {r}"
)
def _increment_top_level_request_and_spend_metrics(
self,
@ -2898,12 +2905,14 @@ class PrometheusLogger(CustomLogger):
max_budget=max_budget,
)
try:
# Note: Setting check_db_only=True bypasses cache and hits DB on every request,
# causing huge latency increase and CPU spikes. Keep check_db_only=False.
user_info = await get_user_object(
user_id=user_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
user_id_upsert=False,
check_db_only=True,
check_db_only=False,
)
except Exception as e:
verbose_logger.debug(

4
litellm/litellm_core_utils/core_helpers.py
View file

@ -94,8 +94,8 @@ def map_finish_reason(
return "length"
elif finish_reason == "tool_use": # anthropic
return "tool_calls"
elif finish_reason == "content_filtered":
return "content_filter"
elif finish_reason == "compaction":
return "length"
return finish_reason

82
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -1,19 +1,48 @@
from typing import Optional
# Pre-define optional kwargs keys as frozenset for O(1) lookups
# These are extracted from kwargs only if present, avoiding unnecessary .get() calls
_OPTIONAL_KWARGS_KEYS = frozenset({
"azure_ad_token",
"tenant_id",
"client_id",
"client_secret",
"azure_username",
"azure_password",
"azure_scope",
"timeout",
"bucket_name",
"vertex_credentials",
"vertex_project",
"vertex_location",
"vertex_ai_project",
"vertex_ai_location",
"vertex_ai_credentials",
"aws_region_name",
"aws_access_key_id",
"aws_secret_access_key",
"aws_session_token",
"aws_session_name",
"aws_profile_name",
"aws_role_name",
"aws_web_identity_token",
"aws_sts_endpoint",
"aws_external_id",
"aws_bedrock_runtime_endpoint",
"tpm",
"rpm",
})
def _get_base_model_from_litellm_call_metadata(
metadata: Optional[dict],
) -> Optional[str]:
if metadata is None:
return None
if metadata is not None:
model_info = metadata.get("model_info", {})
if model_info is not None:
base_model = model_info.get("base_model", None)
if base_model is not None:
return base_model
model_info = metadata.get("model_info")
if model_info:
return model_info.get("base_model")
return None
@ -66,6 +95,7 @@ def get_litellm_params(
litellm_request_debug: Optional[bool] = None,
**kwargs,
) -> dict:
# Build base dict with explicit parameters (always included)
litellm_params = {
"acompletion": acompletion,
"api_key": api_key,
@ -112,37 +142,15 @@ def get_litellm_params(
"ssl_verify": ssl_verify,
"merge_reasoning_content_in_choices": merge_reasoning_content_in_choices,
"api_version": api_version,
"azure_ad_token": kwargs.get("azure_ad_token"),
"tenant_id": kwargs.get("tenant_id"),
"client_id": kwargs.get("client_id"),
"client_secret": kwargs.get("client_secret"),
"azure_username": kwargs.get("azure_username"),
"azure_password": kwargs.get("azure_password"),
"azure_scope": kwargs.get("azure_scope"),
"max_retries": max_retries,
"timeout": kwargs.get("timeout"),
"bucket_name": kwargs.get("bucket_name"),
"vertex_credentials": kwargs.get("vertex_credentials"),
"vertex_project": kwargs.get("vertex_project"),
"vertex_location": kwargs.get("vertex_location"),
"vertex_ai_project": kwargs.get("vertex_ai_project"),
"vertex_ai_location": kwargs.get("vertex_ai_location"),
"vertex_ai_credentials": kwargs.get("vertex_ai_credentials"),
"use_litellm_proxy": use_litellm_proxy,
"litellm_request_debug": litellm_request_debug,
"aws_region_name": kwargs.get("aws_region_name"),
# AWS credentials for Bedrock/Sagemaker
"aws_access_key_id": kwargs.get("aws_access_key_id"),
"aws_secret_access_key": kwargs.get("aws_secret_access_key"),
"aws_session_token": kwargs.get("aws_session_token"),
"aws_session_name": kwargs.get("aws_session_name"),
"aws_profile_name": kwargs.get("aws_profile_name"),
"aws_role_name": kwargs.get("aws_role_name"),
"aws_web_identity_token": kwargs.get("aws_web_identity_token"),
"aws_sts_endpoint": kwargs.get("aws_sts_endpoint"),
"aws_external_id": kwargs.get("aws_external_id"),
"aws_bedrock_runtime_endpoint": kwargs.get("aws_bedrock_runtime_endpoint"),
"tpm": kwargs.get("tpm"),
"rpm": kwargs.get("rpm"),
}
# Sparse extraction: only add kwargs keys that are actually present
if kwargs:
for key in _OPTIONAL_KWARGS_KEYS:
if key in kwargs:
litellm_params[key] = kwargs[key]
return litellm_params

54
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -1,8 +1,35 @@
from typing import Dict, Optional
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import StandardCallbackDynamicParams
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
_supported_callback_params = [
"langfuse_public_key",
"langfuse_secret",
"langfuse_secret_key",
"langfuse_host",
"langfuse_prompt_version",
"gcs_bucket_name",
"gcs_path_service_account",
"langsmith_api_key",
"langsmith_project",
"langsmith_base_url",
"langsmith_sampling_rate",
"langsmith_tenant_id",
"humanloop_api_key",
"arize_api_key",
"arize_space_key",
"arize_space_id",
"posthog_api_key",
"posthog_host",
"braintrust_api_key",
"braintrust_project",
"braintrust_host",
"slack_webhook_url",
"lunary_public_key",
"turn_off_message_logging",
]
def initialize_standard_callback_dynamic_params(
kwargs: Optional[Dict] = None,
@ -15,13 +42,10 @@ def initialize_standard_callback_dynamic_params(
standard_callback_dynamic_params = StandardCallbackDynamicParams()
if kwargs:
_supported_callback_params = (
StandardCallbackDynamicParams.__annotations__.keys()
)
# 1. Check top-level kwargs
for param in _supported_callback_params:
if param in kwargs:
_param_value = kwargs.pop(param)
_param_value = kwargs.get(param)
if (
_param_value is not None
and isinstance(_param_value, str)
@ -30,4 +54,22 @@ def initialize_standard_callback_dynamic_params(
_param_value = get_secret_str(secret_name=_param_value)
standard_callback_dynamic_params[param] = _param_value # type: ignore
# 2. Fallback: check "metadata" or "litellm_params" -> "metadata"
metadata = (kwargs.get("metadata") or {}).copy()
litellm_params = kwargs.get("litellm_params") or {}
if isinstance(litellm_params, dict):
metadata.update(litellm_params.get("metadata") or {})
if isinstance(metadata, dict):
for param in _supported_callback_params:
if param not in standard_callback_dynamic_params and param in metadata:
_param_value = metadata.get(param)
if (
_param_value is not None
and isinstance(_param_value, str)
and "os.environ/" in _param_value
):
_param_value = get_secret_str(secret_name=_param_value)
standard_callback_dynamic_params[param] = _param_value # type: ignore
return standard_callback_dynamic_params

38
litellm/litellm_core_utils/litellm_logging.py
View file

@ -203,6 +203,10 @@ except Exception as e:
EnterpriseStandardLoggingPayloadSetupVAR = None
_in_memory_loggers: List[Any] = []
_STANDARD_LOGGING_METADATA_KEYS: frozenset = frozenset(
StandardLoggingMetadata.__annotations__.keys()
)
### GLOBAL VARIABLES ###
# Cache custom pricing keys as frozenset for O(1) lookups instead of looping through 49 keys
@ -522,7 +526,8 @@ class Logging(LiteLLMLoggingBaseClass):
}
self.litellm_request_debug = litellm_params.get("litellm_request_debug", False)
self.logger_fn = litellm_params.get("logger_fn", None)
verbose_logger.debug(f"self.optional_params: {self.optional_params}")
if _is_debugging_on() or self.litellm_request_debug:
verbose_logger.debug(f"self.optional_params: {self.optional_params}")
self.model_call_details.update(
{
@ -3917,18 +3922,6 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
return langfuse_logger # type: ignore
elif logging_integration == "langfuse_otel":
from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
from litellm.integrations.opentelemetry import (
OpenTelemetry,
OpenTelemetryConfig,
)
langfuse_otel_config = LangfuseOtelLogger.get_langfuse_otel_config()
# The endpoint and headers are now set as environment variables by get_langfuse_otel_config()
otel_config = OpenTelemetryConfig(
exporter=langfuse_otel_config.protocol,
headers=langfuse_otel_config.otlp_auth_headers,
)
for callback in _in_memory_loggers:
if (
@ -3936,8 +3929,10 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
and callback.callback_name == "langfuse_otel"
):
return callback # type: ignore
# Allow LangfuseOtelLogger to initialize its own config safely
# This prevents startup crashes if LANGFUSE keys are not in env (e.g. for dynamic usage)
_otel_logger = LangfuseOtelLogger(
config=otel_config, callback_name="langfuse_otel"
config=None, callback_name="langfuse_otel"
)
_in_memory_loggers.append(_otel_logger)
return _otel_logger # type: ignore
@ -4525,17 +4520,12 @@ class StandardLoggingPayloadSetup:
user_api_key_auth_metadata=None,
)
if isinstance(metadata, dict):
# Filter the metadata dictionary to include only the specified keys
supported_keys = StandardLoggingMetadata.__annotations__.keys()
for key in supported_keys:
if key in metadata:
clean_metadata[key] = metadata[key] # type: ignore
for key in metadata.keys() & _STANDARD_LOGGING_METADATA_KEYS:
clean_metadata[key] = metadata[key] # type: ignore
if metadata.get("user_api_key") is not None:
if is_valid_sha256_hash(str(metadata.get("user_api_key"))):
clean_metadata["user_api_key_hash"] = metadata.get(
"user_api_key"
) # this is the hash
user_api_key = metadata.get("user_api_key")
if user_api_key and isinstance(user_api_key, str) and is_valid_sha256_hash(user_api_key):
clean_metadata["user_api_key_hash"] = user_api_key
_potential_requester_metadata = metadata.get(
"metadata", None
) # check if user passed metadata in the sdk request - e.g. metadata for langsmith logging - https://docs.litellm.ai/docs/observability/langsmith_integration#set-langsmith-fields

10
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -2190,6 +2190,16 @@ def anthropic_messages_pt( # noqa: PLR0915
while msg_i < len(messages) and messages[msg_i]["role"] == "assistant":
assistant_content_block: ChatCompletionAssistantMessage = messages[msg_i] # type: ignore
# Extract compaction_blocks from provider_specific_fields and add them first
_provider_specific_fields_raw = assistant_content_block.get(
"provider_specific_fields"
)
if isinstance(_provider_specific_fields_raw, dict):
_compaction_blocks = _provider_specific_fields_raw.get("compaction_blocks")
if _compaction_blocks and isinstance(_compaction_blocks, list):
# Add compaction blocks at the beginning of assistant content : https://platform.claude.com/docs/en/build-with-claude/compaction
assistant_content.extend(_compaction_blocks) # type: ignore
thinking_blocks = assistant_content_block.get("thinking_blocks", None)
if (
thinking_blocks is not None

47
litellm/litellm_core_utils/redact_messages.py
View file

@ -130,6 +130,11 @@ def perform_redaction(model_call_details: dict, result):
def should_redact_message_logging(model_call_details: dict) -> bool:
"""
Determine if message logging should be redacted.
Priority order:
1. Dynamic parameter (turn_off_message_logging in request)
2. Headers (litellm-disable-message-redaction / litellm-enable-message-redaction)
3. Global setting (litellm.turn_off_message_logging)
"""
litellm_params = model_call_details.get("litellm_params", {})
@ -139,36 +144,36 @@ def should_redact_message_logging(model_call_details: dict) -> bool:
# Get headers from the metadata
request_headers = metadata.get("headers", {}) if isinstance(metadata, dict) else {}
possible_request_headers = [
# Check for headers that explicitly control redaction
if request_headers and bool(
request_headers.get("litellm-disable-message-redaction", False)
):
# User explicitly disabled redaction via header
return False
possible_enable_headers = [
"litellm-enable-message-redaction", # old header. maintain backwards compatibility
"x-litellm-enable-message-redaction", # new header
]
is_redaction_enabled_via_header = False
for header in possible_request_headers:
for header in possible_enable_headers:
if bool(request_headers.get(header, False)):
is_redaction_enabled_via_header = True
break
# check if user opted out of logging message/response to callbacks
if (
litellm.turn_off_message_logging is not True
and is_redaction_enabled_via_header is not True
and _get_turn_off_message_logging_from_dynamic_params(model_call_details)
is not True
):
return False
if request_headers and bool(
request_headers.get("litellm-disable-message-redaction", False)
):
return False
# user has OPTED OUT of message redaction
if _get_turn_off_message_logging_from_dynamic_params(model_call_details) is False:
return False
return True
# Priority 1: Check dynamic parameter first (if explicitly set)
dynamic_turn_off = _get_turn_off_message_logging_from_dynamic_params(model_call_details)
if dynamic_turn_off is not None:
# Dynamic parameter is explicitly set, use it
return dynamic_turn_off
# Priority 2: Check if header explicitly enables redaction
if is_redaction_enabled_via_header:
return True
# Priority 3: Fall back to global setting
return litellm.turn_off_message_logging is True
def redact_message_input_output_from_logging(

4
litellm/litellm_core_utils/token_counter.py
View file

@ -706,7 +706,7 @@ def _count_content_list(
if isinstance(c, str):
num_tokens += count_function(c)
elif c["type"] == "text":
num_tokens += count_function(c.get("text", ""))
num_tokens += count_function(str(c.get("text", "")))
elif c["type"] == "image_url":
image_url = c.get("image_url")
num_tokens += _count_image_tokens(
@ -722,7 +722,7 @@ def _count_content_list(
elif c["type"] == "thinking":
# Claude extended thinking content block
# Count the thinking text and skip signature (opaque signature blob)
thinking_text = c.get("thinking", "")
thinking_text = str(c.get("thinking", ""))
if thinking_text:
num_tokens += count_function(thinking_text)
else:

155
litellm/llms/a2a/chat/guardrail_translation/README.md Normal file
View file

@ -0,0 +1,155 @@
# A2A Protocol Guardrail Translation Handler
Handler for processing A2A (Agent-to-Agent) Protocol messages with guardrails.
## Overview
This handler processes A2A JSON-RPC 2.0 input/output by:
1. Extracting text from message parts (`kind: "text"`)
2. Applying guardrails to text content
3. Mapping guardrailed text back to original structure
## A2A Protocol Format
### Input Format (JSON-RPC 2.0)
```json
{
"jsonrpc": "2.0",
"id": "request-id",
"method": "message/send",
"params": {
"message": {
"kind": "message",
"messageId": "...",
"role": "user",
"parts": [
{"kind": "text", "text": "Hello, my SSN is 123-45-6789"}
]
},
"metadata": {
"guardrails": ["block-ssn"]
}
}
}
```
### Output Formats
The handler supports multiple A2A response formats:
**Direct message:**
```json
{
"result": {
"kind": "message",
"parts": [{"kind": "text", "text": "Response text"}]
}
}
```
**Nested message:**
```json
{
"result": {
"message": {
"parts": [{"kind": "text", "text": "Response text"}]
}
}
}
```
**Task with artifacts:**
```json
{
"result": {
"kind": "task",
"artifacts": [
{"parts": [{"kind": "text", "text": "Artifact text"}]}
]
}
}
```
**Task with status message:**
```json
{
"result": {
"kind": "task",
"status": {
"message": {
"parts": [{"kind": "text", "text": "Status message"}]
}
}
}
}
```
**Streaming artifact-update:**
```json
{
"result": {
"kind": "artifact-update",
"artifact": {
"parts": [{"kind": "text", "text": "Streaming text"}]
}
}
}
```
## Usage
The handler is automatically discovered and applied when guardrails are used with A2A endpoints.
### Via LiteLLM Proxy
```bash
curl -X POST 'http://localhost:4000/a2a/my-agent' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer your-api-key' \
-d '{
"jsonrpc": "2.0",
"id": "1",
"method": "message/send",
"params": {
"message": {
"kind": "message",
"messageId": "msg-1",
"role": "user",
"parts": [{"kind": "text", "text": "Hello, my SSN is 123-45-6789"}]
},
"metadata": {
"guardrails": ["block-ssn"]
}
}
}'
```
### Specifying Guardrails
Guardrails can be specified in the A2A request via the `metadata.guardrails` field:
```json
{
"params": {
"message": {...},
"metadata": {
"guardrails": ["block-ssn", "pii-filter"]
}
}
}
```
## Extension
Override these methods to customize behavior:
- `_extract_texts_from_result()`: Custom text extraction from A2A responses
- `_extract_texts_from_parts()`: Custom text extraction from message parts
- `_apply_text_to_path()`: Custom application of guardrailed text
## Call Types
This handler is registered for:
- `CallTypes.send_message`: Synchronous A2A message sending
- `CallTypes.asend_message`: Asynchronous A2A message sending

11
litellm/llms/a2a/chat/guardrail_translation/__init__.py Normal file
View file

@ -0,0 +1,11 @@
"""A2A Protocol handler for Unified Guardrails."""
from litellm.llms.a2a.chat.guardrail_translation.handler import A2AGuardrailHandler
from litellm.types.utils import CallTypes
guardrail_translation_mappings = {
CallTypes.send_message: A2AGuardrailHandler,
CallTypes.asend_message: A2AGuardrailHandler,
}
__all__ = ["guardrail_translation_mappings"]

315
litellm/llms/a2a/chat/guardrail_translation/handler.py Normal file
View file

@ -0,0 +1,315 @@
"""
A2A Protocol Handler for Unified Guardrails
This module provides guardrail translation support for A2A (Agent-to-Agent) Protocol.
It handles both JSON-RPC 2.0 input requests and output responses, extracting text
from message parts and applying guardrails.
A2A Protocol Format:
- Input: JSON-RPC 2.0 with params.message.parts containing text parts
- Output: JSON-RPC 2.0 with result containing message/artifact parts
"""
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import UserAPIKeyAuth
class A2AGuardrailHandler(BaseTranslation):
"""
Handler for processing A2A Protocol messages with guardrails.
This class provides methods to:
1. Process input messages (pre-call hook) - extracts text from A2A message parts
2. Process output responses (post-call hook) - extracts text from A2A response parts
A2A Message Format:
- Input: params.message.parts[].text (where kind == "text")
- Output: result.message.parts[].text or result.artifacts[].parts[].text
"""
async def process_input_messages(
self,
data: dict,
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
) -> Any:
"""
Process A2A input messages by applying guardrails to text content.
Extracts text from A2A message parts and applies guardrails.
Args:
data: The A2A JSON-RPC 2.0 request data
guardrail_to_apply: The guardrail instance to apply
litellm_logging_obj: Optional logging object
Returns:
Modified data with guardrails applied to text content
"""
# A2A request format: { "params": { "message": { "parts": [...] } } }
params = data.get("params", {})
message = params.get("message", {})
parts = message.get("parts", [])
if not parts:
verbose_proxy_logger.debug("A2A: No parts in message, skipping guardrail")
return data
texts_to_check: List[str] = []
text_part_indices: List[int] = [] # Track which parts contain text
# Step 1: Extract text from all text parts
for part_idx, part in enumerate(parts):
if part.get("kind") == "text":
text = part.get("text", "")
if text:
texts_to_check.append(text)
text_part_indices.append(part_idx)
# Step 2: Apply guardrail to all texts in batch
if texts_to_check:
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
# Pass the structured A2A message to guardrails
inputs["structured_messages"] = [message]
# Include agent model info if available
model = data.get("model")
if model:
inputs["model"] = model
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
input_type="request",
logging_obj=litellm_logging_obj,
)
guardrailed_texts = guardrailed_inputs.get("texts", [])
# Step 3: Apply guardrailed text back to original parts
if guardrailed_texts and len(guardrailed_texts) == len(text_part_indices):
for task_idx, part_idx in enumerate(text_part_indices):
parts[part_idx]["text"] = guardrailed_texts[task_idx]
verbose_proxy_logger.debug("A2A: Processed input message: %s", message)
return data
async def process_output_response(
self,
response: Any,
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
) -> Any:
"""
Process A2A output response by applying guardrails to text content.
Handles multiple A2A response formats:
- Direct message: {"result": {"kind": "message", "parts": [...]}}
- Nested message: {"result": {"message": {"parts": [...]}}}
- Task with artifacts: {"result": {"kind": "task", "artifacts": [{"parts": [...]}]}}
- Task with status message: {"result": {"kind": "task", "status": {"message": {"parts": [...]}}}}
Args:
response: A2A JSON-RPC 2.0 response dict or object
guardrail_to_apply: The guardrail instance to apply
litellm_logging_obj: Optional logging object
user_api_key_dict: User API key metadata
Returns:
Modified response with guardrails applied to text content
"""
# Handle both dict and Pydantic model responses
if hasattr(response, "model_dump"):
response_dict = response.model_dump()
is_pydantic = True
elif isinstance(response, dict):
response_dict = response
is_pydantic = False
else:
verbose_proxy_logger.warning(
"A2A: Unknown response type %s, skipping guardrail", type(response)
)
return response
result = response_dict.get("result", {})
if not result or not isinstance(result, dict):
verbose_proxy_logger.debug("A2A: No result in response, skipping guardrail")
return response
# Find all text-containing parts in the response
texts_to_check: List[str] = []
# Each mapping is (path_to_parts_list, part_index)
# path_to_parts_list is a tuple of keys to navigate to the parts list
task_mappings: List[Tuple[Tuple[str, ...], int]] = []
# Extract texts from all possible locations
self._extract_texts_from_result(
result=result,
texts_to_check=texts_to_check,
task_mappings=task_mappings,
)
if not texts_to_check:
verbose_proxy_logger.debug("A2A: No text content in response")
return response
# Step 2: Apply guardrail to all texts in batch
# Create a request_data dict with response info and user API key metadata
request_data: dict = {"response": response_dict}
# Add user API key metadata with prefixed keys
user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
if user_metadata:
request_data["litellm_metadata"] = user_metadata
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=request_data,
input_type="response",
logging_obj=litellm_logging_obj,
)
guardrailed_texts = guardrailed_inputs.get("texts", [])
# Step 3: Apply guardrailed text back to original response
if guardrailed_texts and len(guardrailed_texts) == len(task_mappings):
for task_idx, (path, part_idx) in enumerate(task_mappings):
self._apply_text_to_path(
result=result,
path=path,
part_idx=part_idx,
text=guardrailed_texts[task_idx],
)
verbose_proxy_logger.debug("A2A: Processed output response")
# Update the original response
if is_pydantic:
# For Pydantic models, we need to update the underlying dict
# and the model will reflect the changes
response_dict["result"] = result
return response
else:
response["result"] = result
return response
def _extract_texts_from_result(
self,
result: Dict[str, Any],
texts_to_check: List[str],
task_mappings: List[Tuple[Tuple[str, ...], int]],
) -> None:
"""
Extract text from all possible locations in an A2A result.
Handles multiple response formats:
1. Direct message with parts: {"parts": [...]}
2. Nested message: {"message": {"parts": [...]}}
3. Task with artifacts: {"artifacts": [{"parts": [...]}]}
4. Task with status message: {"status": {"message": {"parts": [...]}}}
5. Streaming artifact-update: {"artifact": {"parts": [...]}}
"""
# Case 1: Direct parts in result (direct message)
if "parts" in result:
self._extract_texts_from_parts(
parts=result["parts"],
path=("parts",),
texts_to_check=texts_to_check,
task_mappings=task_mappings,
)
# Case 2: Nested message
message = result.get("message")
if message and isinstance(message, dict) and "parts" in message:
self._extract_texts_from_parts(
parts=message["parts"],
path=("message", "parts"),
texts_to_check=texts_to_check,
task_mappings=task_mappings,
)
# Case 3: Streaming artifact-update (singular artifact)
artifact = result.get("artifact")
if artifact and isinstance(artifact, dict) and "parts" in artifact:
self._extract_texts_from_parts(
parts=artifact["parts"],
path=("artifact", "parts"),
texts_to_check=texts_to_check,
task_mappings=task_mappings,
)
# Case 4: Task with status message
status = result.get("status", {})
if isinstance(status, dict):
status_message = status.get("message")
if (
status_message
and isinstance(status_message, dict)
and "parts" in status_message
):
self._extract_texts_from_parts(
parts=status_message["parts"],
path=("status", "message", "parts"),
texts_to_check=texts_to_check,
task_mappings=task_mappings,
)
# Case 5: Task with artifacts (plural, array)
artifacts = result.get("artifacts", [])
if artifacts and isinstance(artifacts, list):
for artifact_idx, art in enumerate(artifacts):
if isinstance(art, dict) and "parts" in art:
self._extract_texts_from_parts(
parts=art["parts"],
path=("artifacts", str(artifact_idx), "parts"),
texts_to_check=texts_to_check,
task_mappings=task_mappings,
)
def _extract_texts_from_parts(
self,
parts: List[Dict[str, Any]],
path: Tuple[str, ...],
texts_to_check: List[str],
task_mappings: List[Tuple[Tuple[str, ...], int]],
) -> None:
"""Extract text from message parts."""
for part_idx, part in enumerate(parts):
if part.get("kind") == "text":
text = part.get("text", "")
if text:
texts_to_check.append(text)
task_mappings.append((path, part_idx))
def _apply_text_to_path(
self,
result: Dict[Union[str, int], Any],
path: Tuple[str, ...],
part_idx: int,
text: str,
) -> None:
"""Apply guardrailed text back to the specified path in the result."""
# Navigate to the parts list
current = result
for key in path:
if key.isdigit():
# Array index
current = current[int(key)]
else:
current = current[key]
# Update the text in the part
current[part_idx]["text"] = text

23
litellm/llms/anthropic/chat/handler.py
View file

@ -512,6 +512,9 @@ class ModelResponseIterator:
# Accumulate web_search_tool_result blocks for multi-turn reconstruction
# See: https://github.com/BerriAI/litellm/issues/17737
self.web_search_results: List[Dict[str, Any]] = []
# Accumulate compaction blocks for multi-turn reconstruction
self.compaction_blocks: List[Dict[str, Any]] = []
def check_empty_tool_call_args(self) -> bool:
"""
@ -592,6 +595,12 @@ class ModelResponseIterator:
)
]
provider_specific_fields["thinking_blocks"] = thinking_blocks
elif "content" in content_block["delta"] and content_block["delta"].get("type") == "compaction_delta":
# Handle compaction delta
provider_specific_fields["compaction_delta"] = {
"type": "compaction_delta",
"content": content_block["delta"]["content"]
}
return text, tool_use, thinking_blocks, provider_specific_fields
@ -721,6 +730,20 @@ class ModelResponseIterator:
provider_specific_fields=provider_specific_fields,
)
elif content_block_start["content_block"]["type"] == "compaction":
# Handle compaction blocks
# The full content comes in content_block_start
self.compaction_blocks.append(
content_block_start["content_block"]
)
provider_specific_fields["compaction_blocks"] = (
self.compaction_blocks
)
provider_specific_fields["compaction_start"] = {
"type": "compaction",
"content": content_block_start["content_block"].get("content", "")
}
elif content_block_start["content_block"]["type"].endswith("_tool_result"):
# Handle all tool result types (web_search, bash_code_execution, text_editor, etc.)
content_type = content_block_start["content_block"]["type"]

133
litellm/llms/anthropic/chat/transformation.py
View file

@ -170,9 +170,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
tool_call["caller"] = cast(Dict[str, Any], anthropic_tool_content["caller"]) # type: ignore[typeddict-item]
return tool_call
def _is_claude_opus_4_5(self, model: str) -> bool:
@staticmethod
def _is_claude_opus_4_6(model: str) -> bool:
"""Check if the model is Claude Opus 4.5."""
return "opus-4-5" in model.lower() or "opus_4_5" in model.lower()
return "opus-4-6" in model.lower() or "opus_4_6" in model.lower()
def get_supported_openai_params(self, model: str):
params = [
@ -659,32 +660,38 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
@staticmethod
def _map_reasoning_effort(
reasoning_effort: Optional[Union[REASONING_EFFORT, str]],
reasoning_effort: Optional[Union[REASONING_EFFORT, str]],
model: str,
) -> Optional[AnthropicThinkingParam]:
if reasoning_effort is None:
return None
elif reasoning_effort == "low":
if AnthropicConfig._is_claude_opus_4_6(model):
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
)
elif reasoning_effort == "medium":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
)
elif reasoning_effort == "high":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
)
elif reasoning_effort == "minimal":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET,
type="adaptive",
)
else:
raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}")
if reasoning_effort is None:
return None
elif reasoning_effort == "low":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
)
elif reasoning_effort == "medium":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
)
elif reasoning_effort == "high":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
)
elif reasoning_effort == "minimal":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET,
)
else:
raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}")
def _extract_json_schema_from_response_format(
self, value: Optional[dict]
@ -860,13 +867,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if param == "thinking":
optional_params["thinking"] = value
elif param == "reasoning_effort" and isinstance(value, str):
# For Claude Opus 4.5, map reasoning_effort to output_config
if self._is_claude_opus_4_5(model):
optional_params["output_config"] = {"effort": value}
# For other models, map to thinking parameter
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
value
reasoning_effort=value, model=model
)
elif param == "web_search_options" and isinstance(value, dict):
hosted_web_search_tool = self.map_web_search_tool(
@ -877,6 +879,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
elif param == "extra_headers":
optional_params["extra_headers"] = value
elif param == "context_management" and isinstance(value, dict):
# Pass through Anthropic-specific context_management parameter
optional_params["context_management"] = value
## handle thinking tokens
self.update_optional_params_with_thinking_tokens(
@ -1026,9 +1031,37 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if beta_value not in existing_values:
headers["anthropic-beta"] = f"{existing_beta}, {beta_value}"
def _ensure_context_management_beta_header(self, headers: dict) -> None:
beta_value = ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
self._ensure_beta_header(headers, beta_value)
def _ensure_context_management_beta_header(
self, headers: dict, context_management: dict
) -> None:
"""
Add appropriate beta headers based on context_management edits.
- If any edit has type "compact_20260112", add compact-2026-01-12 header
- For all other edits, add context-management-2025-06-27 header
"""
edits = context_management.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
self._ensure_beta_header(
headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value
)
# Add context management header if any other edits exist
if has_other:
self._ensure_beta_header(
headers, ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
)
def update_headers_with_optional_anthropic_beta(
self, headers: dict, optional_params: dict
@ -1056,7 +1089,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers, ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
)
if optional_params.get("context_management") is not None:
self._ensure_context_management_beta_header(headers)
self._ensure_context_management_beta_header(
headers, optional_params["context_management"]
)
if optional_params.get("output_format") is not None:
self._ensure_beta_header(
headers, ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value
@ -1225,6 +1260,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
List[ChatCompletionToolCallChunk],
Optional[List[Any]],
Optional[List[Any]],
Optional[List[Any]],
]:
text_content = ""
citations: Optional[List[Any]] = None
@ -1237,6 +1273,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
tool_calls: List[ChatCompletionToolCallChunk] = []
web_search_results: Optional[List[Any]] = None
tool_results: Optional[List[Any]] = None
compaction_blocks: Optional[List[Any]] = None
for idx, content in enumerate(completion_response["content"]):
if content["type"] == "text":
text_content += content["text"]
@ -1278,6 +1315,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
thinking_blocks.append(
cast(ChatCompletionRedactedThinkingBlock, content)
)
## COMPACTION
elif content["type"] == "compaction":
if compaction_blocks is None:
compaction_blocks = []
compaction_blocks.append(content)
## CITATIONS
if content.get("citations") is not None:
@ -1299,7 +1342,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if thinking_content is not None:
reasoning_content += thinking_content
return text_content, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results
return text_content, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks
def calculate_usage(
self,
@ -1316,6 +1359,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
cache_creation_token_details: Optional[CacheCreationTokenDetails] = None
web_search_requests: Optional[int] = None
tool_search_requests: Optional[int] = None
inference_geo: Optional[str] = None
if "inference_geo" in _usage and _usage["inference_geo"] is not None:
inference_geo = _usage["inference_geo"]
if (
"cache_creation_input_tokens" in _usage
and _usage["cache_creation_input_tokens"] is not None
@ -1399,6 +1446,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if (web_search_requests is not None or tool_search_requests is not None)
else None
),
inference_geo=inference_geo,
)
return usage
@ -1442,6 +1490,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
tool_calls,
web_search_results,
tool_results,
compaction_blocks,
) = self.extract_response_content(completion_response=completion_response)
if (
@ -1469,6 +1518,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
provider_specific_fields["tool_results"] = tool_results
if container is not None:
provider_specific_fields["container"] = container
if compaction_blocks is not None:
provider_specific_fields["compaction_blocks"] = compaction_blocks
_message = litellm.Message(
tool_calls=tool_calls,
@ -1477,6 +1528,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
thinking_blocks=thinking_blocks,
reasoning_content=reasoning_content,
)
_message.provider_specific_fields = provider_specific_fields
## HANDLE JSON MODE - anthropic returns single function call
json_mode_message = self._transform_response_for_json_mode(
@ -1507,18 +1559,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
model_response.created = int(time.time())
model_response.model = completion_response["model"]
context_management_response = completion_response.get("context_management")
if context_management_response is not None:
_hidden_params["context_management"] = context_management_response
try:
model_response.__dict__["context_management"] = (
context_management_response
)
except Exception:
pass
model_response._hidden_params = _hidden_params
return model_response
def get_prefix_prompt(self, messages: List[AllMessageValues]) -> Optional[str]:

11
litellm/llms/anthropic/cost_calculation.py
View file

@ -22,10 +22,17 @@ def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]:
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
"""
return generic_cost_per_token(
model=model, usage=usage, custom_llm_provider="anthropic"
# If usage has inference_geo, prepend it as prefix to model name
if hasattr(usage, "inference_geo") and usage.inference_geo and usage.inference_geo.lower() not in ["global", "not_available"]:
model_with_geo_prefix = f"{usage.inference_geo}/{model}"
else:
model_with_geo_prefix = model
prompt_cost, completion_cost = generic_cost_per_token(
model=model_with_geo_prefix, usage=usage, custom_llm_provider="anthropic"
)
return prompt_cost, completion_cost
def get_cost_for_anthropic_web_search(
model_info: Optional["ModelInfo"] = None,

31
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -2,6 +2,9 @@ from typing import Any, AsyncIterator, Dict, List, Optional, Tuple
import httpx
from litellm.anthropic_beta_headers_manager import (
update_headers_with_filtered_beta,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import verbose_logger
from litellm.llms.base_llm.anthropic_messages.transformation import (
@ -90,6 +93,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
optional_params=optional_params,
)
headers = update_headers_with_filtered_beta(
headers=headers,
provider="anthropic",
)
return headers, api_base
def transform_anthropic_messages_request(
@ -189,8 +197,27 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
beta_values.update(b.strip() for b in existing_beta.split(","))
# Check for context management
if optional_params.get("context_management") is not None:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
context_management_param = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits = context_management_param.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
# Check for structured outputs
if optional_params.get("output_format") is not None:

9
litellm/llms/azure_ai/anthropic/transformation.py
View file

@ -3,6 +3,9 @@ Azure Anthropic transformation config - extends AnthropicConfig with Azure authe
"""
from typing import TYPE_CHECKING, Dict, List, Optional, Union
from litellm.anthropic_beta_headers_manager import (
update_headers_with_filtered_beta,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.types.llms.openai import AllMessageValues
@ -87,6 +90,12 @@ class AzureAnthropicConfig(AnthropicConfig):
if "anthropic-version" not in headers:
headers["anthropic-version"] = "2023-06-01"
# Filter out unsupported beta headers for Azure AI
headers = update_headers_with_filtered_beta(
headers=headers,
provider="azure_ai",
)
return headers
def transform_request(

161
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -11,6 +11,9 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.anthropic_beta_headers_manager import (
filter_and_transform_beta_headers,
)
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.core_helpers import (
filter_exceptions_from_params,
@ -66,6 +69,7 @@ from ..common_utils import (
BedrockModelInfo,
get_anthropic_beta_from_headers,
get_bedrock_tool_name,
is_claude_4_5_on_bedrock,
)
# Computer use tool prefixes supported by Bedrock
@ -81,6 +85,7 @@ BEDROCK_COMPUTER_USE_TOOLS = [
UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS = [
"advanced-tool-use", # Bedrock Converse doesn't support advanced-tool-use beta headers
"prompt-caching", # Prompt caching not supported in Converse API
"compact-2026-01-12", # The compact beta feature is not currently supported on the Converse and ConverseStream APIs
]
@ -306,9 +311,7 @@ class AmazonConverseConfig(BaseConfig):
return "nova-2-lite" in model_without_region
def _map_web_search_options(
self,
web_search_options: dict,
model: str
self, web_search_options: dict, model: str
) -> Optional[BedrockToolBlock]:
"""
Map web_search_options to Nova grounding systemTool.
@ -431,7 +434,7 @@ class AmazonConverseConfig(BaseConfig):
else:
# Anthropic and other models: convert to thinking parameter
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
reasoning_effort
reasoning_effort=reasoning_effort, model=model
)
def get_supported_openai_params(self, model: str) -> List[str]:
@ -617,37 +620,6 @@ class AmazonConverseConfig(BaseConfig):
return transformed_tools
def _filter_unsupported_beta_headers_for_bedrock(
self, model: str, beta_list: list
) -> list:
"""
Remove beta headers that are not supported on Bedrock Converse API for the given model.
Extended thinking beta headers are only supported on specific Claude 4+ models.
Some beta headers are universally unsupported on Bedrock Converse API.
Args:
model: The model name
beta_list: The list of beta headers to filter
Returns:
Filtered list of beta headers
"""
filtered_betas = []
# 1. Filter out beta headers that are universally unsupported on Bedrock Converse
for beta in beta_list:
should_keep = True
for unsupported_pattern in UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS:
if unsupported_pattern in beta.lower():
should_keep = False
break
if should_keep:
filtered_betas.append(beta)
return filtered_betas
def _separate_computer_use_tools(
self, tools: List[OpenAIChatCompletionToolParam], model: str
) -> Tuple[
@ -808,11 +780,11 @@ class AmazonConverseConfig(BaseConfig):
if param == "web_search_options" and isinstance(value, dict):
# Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)`
# because empty dict {} is falsy but is a valid way to enable Nova grounding
grounding_tool = self._map_web_search_options(value, model)
if grounding_tool is not None:
optional_params = self._add_tools_to_optional_params(
optional_params=optional_params, tools=[grounding_tool]
)
grounding_tool = self._map_web_search_options(value, model)
if grounding_tool is not None:
optional_params = self._add_tools_to_optional_params(
optional_params=optional_params, tools=[grounding_tool]
)
# Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models
# Nova Lite 2 handles token budgeting differently through reasoningConfig
@ -926,6 +898,7 @@ class AmazonConverseConfig(BaseConfig):
ChatCompletionAssistantMessage,
],
block_type: Literal["system"],
model: Optional[str] = None,
) -> Optional[SystemContentBlock]:
pass
@ -939,6 +912,7 @@ class AmazonConverseConfig(BaseConfig):
ChatCompletionAssistantMessage,
],
block_type: Literal["content_block"],
model: Optional[str] = None,
) -> Optional[ContentBlock]:
pass
@ -951,16 +925,26 @@ class AmazonConverseConfig(BaseConfig):
ChatCompletionAssistantMessage,
],
block_type: Literal["system", "content_block"],
model: Optional[str] = None,
) -> Optional[Union[SystemContentBlock, ContentBlock]]:
if message_block.get("cache_control", None) is None:
cache_control = message_block.get("cache_control", None)
if cache_control is None:
return None
cache_point = CachePointBlock(type="default")
if isinstance(cache_control, dict) and "ttl" in cache_control:
ttl = cache_control["ttl"]
if ttl in ["5m", "1h"] and model is not None:
if is_claude_4_5_on_bedrock(model):
cache_point["ttl"] = ttl
if block_type == "system":
return SystemContentBlock(cachePoint=CachePointBlock(type="default"))
return SystemContentBlock(cachePoint=cache_point)
else:
return ContentBlock(cachePoint=CachePointBlock(type="default"))
return ContentBlock(cachePoint=cache_point)
def _transform_system_message(
self, messages: List[AllMessageValues]
self, messages: List[AllMessageValues], model: Optional[str] = None
) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]:
system_prompt_indices = []
system_content_blocks: List[SystemContentBlock] = []
@ -972,7 +956,7 @@ class AmazonConverseConfig(BaseConfig):
SystemContentBlock(text=message["content"])
)
cache_block = self._get_cache_point_block(
message, block_type="system"
message, block_type="system", model=model
)
if cache_block:
system_content_blocks.append(cache_block)
@ -983,7 +967,7 @@ class AmazonConverseConfig(BaseConfig):
SystemContentBlock(text=m["text"])
)
cache_block = self._get_cache_point_block(
m, block_type="system"
m, block_type="system", model=model
)
if cache_block:
system_content_blocks.append(cache_block)
@ -1112,7 +1096,28 @@ class AmazonConverseConfig(BaseConfig):
# Add computer use tools and anthropic_beta if needed (only when computer use tools are present)
if computer_use_tools:
anthropic_beta_list.append("computer-use-2024-10-22")
# Determine the correct computer-use beta header based on model
# "computer-use-2025-11-24" for Claude Opus 4.6, Claude Opus 4.5
# "computer-use-2025-01-24" for Claude Sonnet 4.5, Haiku 4.5, Opus 4.1, Sonnet 4, Opus 4, and Sonnet 3.7
# "computer-use-2024-10-22" for older models
model_lower = model.lower()
if "opus-4.6" in model_lower or "opus_4.6" in model_lower or "opus-4-6" in model_lower or "opus_4_6" in model_lower:
computer_use_header = "computer-use-2025-11-24"
elif "opus-4.5" in model_lower or "opus_4.5" in model_lower or "opus-4-5" in model_lower or "opus_4_5" in model_lower:
computer_use_header = "computer-use-2025-11-24"
elif any(pattern in model_lower for pattern in [
"sonnet-4.5", "sonnet_4.5", "sonnet-4-5", "sonnet_4_5",
"haiku-4.5", "haiku_4.5", "haiku-4-5", "haiku_4_5",
"opus-4.1", "opus_4.1", "opus-4-1", "opus_4_1",
"sonnet-4", "sonnet_4",
"opus-4", "opus_4",
"sonnet-3.7", "sonnet_3.7", "sonnet-3-7", "sonnet_3_7"
]):
computer_use_header = "computer-use-2025-01-24"
else:
computer_use_header = "computer-use-2024-10-22"
anthropic_beta_list.append(computer_use_header)
# Transform computer use tools to proper Bedrock format
transformed_computer_tools = self._transform_computer_use_tools(
computer_use_tools
@ -1137,14 +1142,14 @@ class AmazonConverseConfig(BaseConfig):
if beta not in seen:
unique_betas.append(beta)
seen.add(beta)
# Filter out unsupported beta headers for Bedrock Converse API
filtered_betas = self._filter_unsupported_beta_headers_for_bedrock(
model=model,
beta_list=unique_betas,
filtered_betas = filter_and_transform_beta_headers(
beta_headers=unique_betas,
provider="bedrock_converse",
)
additional_request_params["anthropic_beta"] = filtered_betas
if filtered_betas:
additional_request_params["anthropic_beta"] = filtered_betas
return bedrock_tools, anthropic_beta_list
@ -1196,9 +1201,11 @@ class AmazonConverseConfig(BaseConfig):
)
# Prepare and separate parameters
inference_params, additional_request_params, request_metadata = self._prepare_request_params(
optional_params, model
)
(
inference_params,
additional_request_params,
request_metadata,
) = self._prepare_request_params(optional_params, model)
original_tools = inference_params.pop("tools", [])
@ -1250,7 +1257,9 @@ class AmazonConverseConfig(BaseConfig):
litellm_params: dict,
headers: Optional[dict] = None,
) -> RequestObject:
messages, system_content_blocks = self._transform_system_message(messages)
messages, system_content_blocks = self._transform_system_message(
messages, model=model
)
# Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text(
@ -1306,7 +1315,9 @@ class AmazonConverseConfig(BaseConfig):
litellm_params: dict,
headers: Optional[dict] = None,
) -> RequestObject:
messages, system_content_blocks = self._transform_system_message(messages)
messages, system_content_blocks = self._transform_system_message(
messages, model=model
)
# Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text(
@ -1484,7 +1495,9 @@ class AmazonConverseConfig(BaseConfig):
return message, returned_finish_reason
def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[
def _translate_message_content(
self, content_blocks: List[ContentBlock]
) -> Tuple[
str,
List[ChatCompletionToolCallChunk],
Optional[List[BedrockConverseReasoningContentBlock]],
@ -1501,9 +1514,9 @@ class AmazonConverseConfig(BaseConfig):
"""
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
reasoningContentBlocks: Optional[
List[BedrockConverseReasoningContentBlock]
] = None
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
for idx, content in enumerate(content_blocks):
"""
@ -1557,7 +1570,7 @@ class AmazonConverseConfig(BaseConfig):
return content_str, tools, reasoningContentBlocks, citationsContentBlocks
def _transform_response( # noqa: PLR0915
def _transform_response( # noqa: PLR0915
self,
model: str,
response: httpx.Response,
@ -1630,9 +1643,9 @@ class AmazonConverseConfig(BaseConfig):
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
reasoningContentBlocks: Optional[
List[BedrockConverseReasoningContentBlock]
] = None
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
if message is not None:
@ -1651,15 +1664,17 @@ class AmazonConverseConfig(BaseConfig):
provider_specific_fields["citationsContent"] = citationsContentBlocks
if provider_specific_fields:
chat_completion_message["provider_specific_fields"] = provider_specific_fields
chat_completion_message[
"provider_specific_fields"
] = provider_specific_fields
if reasoningContentBlocks is not None:
chat_completion_message["reasoning_content"] = (
self._transform_reasoning_content(reasoningContentBlocks)
)
chat_completion_message["thinking_blocks"] = (
self._transform_thinking_blocks(reasoningContentBlocks)
)
chat_completion_message[
"reasoning_content"
] = self._transform_reasoning_content(reasoningContentBlocks)
chat_completion_message[
"thinking_blocks"
] = self._transform_thinking_blocks(reasoningContentBlocks)
chat_completion_message["content"] = content_str
if (
json_mode is True

33
litellm/llms/bedrock/common_utils.py
View file

@ -446,6 +446,29 @@ def get_bedrock_base_model(model: str) -> str:
return model
def is_claude_4_5_on_bedrock(model: str) -> bool:
"""
Check if the model is a Claude 4.5 model on Bedrock.
Claude 4.5 models support prompt caching with '5m' and '1h' TTL on Bedrock.
"""
model_lower = model.lower()
claude_4_5_patterns = [
"sonnet-4.5",
"sonnet_4.5",
"sonnet-4-5",
"sonnet_4_5",
"haiku-4.5",
"haiku_4.5",
"haiku-4-5",
"haiku_4_5",
"opus-4.5",
"opus_4.5",
"opus-4-5",
"opus_4_5",
]
return any(pattern in model_lower for pattern in claude_4_5_patterns)
# Import after standalone functions to avoid circular imports
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
@ -815,21 +838,23 @@ def get_anthropic_beta_from_headers(headers: dict) -> List[str]:
# If it's already a list, return it
if isinstance(anthropic_beta_header, list):
return anthropic_beta_header
# Try to parse as JSON array first (e.g., '["interleaved-thinking-2025-05-14", "claude-code-20250219"]')
if isinstance(anthropic_beta_header, str):
anthropic_beta_header = anthropic_beta_header.strip()
if anthropic_beta_header.startswith("[") and anthropic_beta_header.endswith("]"):
if anthropic_beta_header.startswith("[") and anthropic_beta_header.endswith(
"]"
):
try:
parsed = json.loads(anthropic_beta_header)
if isinstance(parsed, list):
return [str(beta).strip() for beta in parsed]
except json.JSONDecodeError:
pass # Fall through to comma-separated parsing
# Fall back to comma-separated values
return [beta.strip() for beta in anthropic_beta_header.split(",")]
return []

184
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -12,6 +12,9 @@ from typing import (
import httpx
from litellm.anthropic_beta_headers_manager import (
filter_and_transform_beta_headers,
)
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
@ -23,7 +26,10 @@ from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
)
from litellm.llms.bedrock.common_utils import get_anthropic_beta_from_headers
from litellm.llms.bedrock.common_utils import (
get_anthropic_beta_from_headers,
is_claude_4_5_on_bedrock,
)
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
@ -52,10 +58,6 @@ class AmazonAnthropicClaudeMessagesConfig(
# Beta header patterns that are not supported by Bedrock Invoke API
# These will be filtered out to prevent 400 "invalid beta flag" errors
UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS = [
"advanced-tool-use", # Bedrock Invoke doesn't support advanced-tool-use beta headers
"prompt-caching-scope"
]
def __init__(self, **kwargs):
BaseAnthropicMessagesConfig.__init__(self, **kwargs)
@ -116,15 +118,22 @@ class AmazonAnthropicClaudeMessagesConfig(
)
def _remove_ttl_from_cache_control(
self, anthropic_messages_request: Dict
self, anthropic_messages_request: Dict, model: Optional[str] = None
) -> None:
"""
Remove `ttl` field from cache_control in messages.
Bedrock doesn't support the ttl field in cache_control.
Update: Bedock supports `5m` and `1h` for Claude 4.5 models.
Args:
anthropic_messages_request: The request dictionary to modify in-place
model: The model name to check if it supports ttl
"""
is_claude_4_5 = False
if model:
is_claude_4_5 = self._is_claude_4_5_on_bedrock(model)
if "messages" in anthropic_messages_request:
for message in anthropic_messages_request["messages"]:
if isinstance(message, dict) and "content" in message:
@ -133,7 +142,14 @@ class AmazonAnthropicClaudeMessagesConfig(
for item in content:
if isinstance(item, dict) and "cache_control" in item:
cache_control = item["cache_control"]
if isinstance(cache_control, dict) and "ttl" in cache_control:
if (
isinstance(cache_control, dict)
and "ttl" in cache_control
):
ttl = cache_control["ttl"]
if is_claude_4_5 and ttl in ["5m", "1h"]:
continue
cache_control.pop("ttl", None)
def _supports_extended_thinking_on_bedrock(self, model: str) -> bool:
@ -155,10 +171,18 @@ class AmazonAnthropicClaudeMessagesConfig(
# Supported models on Bedrock for extended thinking
supported_patterns = [
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5", # Opus 4.5
"opus-4.1", "opus_4.1", "opus-4-1", "opus_4_1", # Opus 4.1
"opus-4", "opus_4", # Opus 4
"sonnet-4", "sonnet_4", # Sonnet 4
"opus-4.5",
"opus_4.5",
"opus-4-5",
"opus_4_5", # Opus 4.5
"opus-4.1",
"opus_4.1",
"opus-4-1",
"opus_4_1", # Opus 4.1
"opus-4",
"opus_4", # Opus 4
"sonnet-4",
"sonnet_4", # Sonnet 4
]
return any(pattern in model_lower for pattern in supported_patterns)
@ -175,10 +199,27 @@ class AmazonAnthropicClaudeMessagesConfig(
"""
model_lower = model.lower()
opus_4_5_patterns = [
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5",
"opus-4.5",
"opus_4.5",
"opus-4-5",
"opus_4_5",
]
return any(pattern in model_lower for pattern in opus_4_5_patterns)
def _is_claude_4_5_on_bedrock(self, model: str) -> bool:
"""
Check if the model is Claude 4.5 on Bedrock.
Claude Sonnet 4.5, Haiku 4.5, and Opus 4.5 support 1-hour prompt caching.
Args:
model: The model name
Returns:
True if the model is Claude 4.5
"""
return is_claude_4_5_on_bedrock(model)
def _supports_tool_search_on_bedrock(self, model: str) -> bool:
"""
Check if the model supports tool search on Bedrock.
@ -199,9 +240,15 @@ class AmazonAnthropicClaudeMessagesConfig(
# Supported models for tool search on Bedrock
supported_patterns = [
# Opus 4.5
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5",
"opus-4.5",
"opus_4.5",
"opus-4-5",
"opus_4_5",
# Sonnet 4.5
"sonnet-4.5", "sonnet_4.5", "sonnet-4-5", "sonnet_4_5",
"sonnet-4.5",
"sonnet_4.5",
"sonnet-4-5",
"sonnet_4_5",
]
return any(pattern in model_lower for pattern in supported_patterns)
@ -228,41 +275,48 @@ class AmazonAnthropicClaudeMessagesConfig(
model: The model name
beta_set: The set of beta headers to filter in-place
"""
beta_headers_to_remove = set()
has_advanced_tool_use = False
# 1. Filter out beta headers that are universally unsupported on Bedrock Invoke and track if advanced-tool-use header is present
for beta in beta_set:
for unsupported_pattern in self.UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS:
if unsupported_pattern in beta.lower():
beta_headers_to_remove.add(beta)
has_advanced_tool_use = True
break
# 1. Handle header transformations BEFORE filtering
# (advanced-tool-use -> tool-search-tool)
# This must happen before filtering because advanced-tool-use is in the unsupported list
has_advanced_tool_use = "advanced-tool-use-2025-11-20" in beta_set
if has_advanced_tool_use and self._supports_tool_search_on_bedrock(model):
beta_set.discard("advanced-tool-use-2025-11-20")
beta_set.add("tool-search-tool-2025-10-19")
beta_set.add("tool-examples-2025-10-29")
# 2. Filter out extended thinking headers for models that don't support them
# 2. Apply provider-level filtering using centralized JSON config
beta_list = list(beta_set)
filtered_list = filter_and_transform_beta_headers(
beta_headers=beta_list,
provider="bedrock",
)
# Update the set with filtered headers
beta_set.clear()
beta_set.update(filtered_list)
# 2.1. Handle model-specific exceptions: structured-outputs is only supported on Opus 4.6
# Re-add structured-outputs if it was in the original set and model is Opus 4.6
model_lower = model.lower()
is_opus_4_6 = any(pattern in model_lower for pattern in ["opus-4.6", "opus_4.6", "opus-4-6", "opus_4_6"])
if is_opus_4_6 and "structured-outputs-2025-11-13" in beta_list:
beta_set.add("structured-outputs-2025-11-13")
# 3. Filter out extended thinking headers for models that don't support them
extended_thinking_patterns = [
"extended-thinking",
"interleaved-thinking",
]
if not self._supports_extended_thinking_on_bedrock(model):
beta_headers_to_remove = set()
for beta in beta_set:
for pattern in extended_thinking_patterns:
if pattern in beta.lower():
beta_headers_to_remove.add(beta)
break
# Remove all filtered headers
for beta in beta_headers_to_remove:
beta_set.discard(beta)
# 3. Translate advanced-tool-use to Bedrock-specific headers for models that support tool search
# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html
# Ref: https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-search-tool
if has_advanced_tool_use and self._supports_tool_search_on_bedrock(model):
beta_set.add("tool-search-tool-2025-10-19")
beta_set.add("tool-examples-2025-10-29")
for beta in beta_headers_to_remove:
beta_set.discard(beta)
def _get_tool_search_beta_header_for_bedrock(
self,
@ -290,7 +344,9 @@ class AmazonAnthropicClaudeMessagesConfig(
input_examples_used: Whether input examples are used
beta_set: The set of beta headers to modify in-place
"""
if tool_search_used and not (programmatic_tool_calling_used or input_examples_used):
if tool_search_used and not (
programmatic_tool_calling_used or input_examples_used
):
beta_set.discard(ANTHROPIC_TOOL_SEARCH_BETA_HEADER)
if "opus-4" in model.lower() or "opus_4" in model.lower():
beta_set.add("tool-search-tool-2025-10-19")
@ -302,13 +358,13 @@ class AmazonAnthropicClaudeMessagesConfig(
) -> None:
"""
Convert Anthropic output_format to inline schema in message content.
Bedrock Invoke doesn't support the output_format parameter, so we embed
the schema directly into the user message content as text instructions.
This approach adds the schema to the last user message, instructing the model
to respond in the specified JSON format.
Args:
output_format: The output_format dict with 'type' and 'schema'
anthropic_messages_request: The request dict to modify in-place
@ -321,35 +377,32 @@ class AmazonAnthropicClaudeMessagesConfig(
schema = output_format.get("schema")
if not schema:
return
# Get messages from the request
messages = anthropic_messages_request.get("messages", [])
if not messages:
return
# Find the last user message
last_user_message_idx = None
for idx in range(len(messages) - 1, -1, -1):
if messages[idx].get("role") == "user":
last_user_message_idx = idx
break
if last_user_message_idx is None:
return
last_user_message = messages[last_user_message_idx]
content = last_user_message.get("content", [])
# Ensure content is a list
if isinstance(content, str):
content = [{"type": "text", "text": content}]
last_user_message["content"] = content
# Add schema as text content to the message
schema_text = {
"type": "text",
"text": json.dumps(schema)
}
schema_text = {"type": "text", "text": json.dumps(schema)}
content.append(schema_text)
def transform_anthropic_messages_request(
@ -374,9 +427,9 @@ class AmazonAnthropicClaudeMessagesConfig(
# 1. anthropic_version is required for all claude models
if "anthropic_version" not in anthropic_messages_request:
anthropic_messages_request["anthropic_version"] = (
self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION
)
anthropic_messages_request[
"anthropic_version"
] = self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION
# 2. `stream` is not allowed in request body for bedrock invoke
if "stream" in anthropic_messages_request:
@ -386,8 +439,10 @@ class AmazonAnthropicClaudeMessagesConfig(
if "model" in anthropic_messages_request:
anthropic_messages_request.pop("model", None)
# 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it)
self._remove_ttl_from_cache_control(anthropic_messages_request)
# 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it for older models)
self._remove_ttl_from_cache_control(
anthropic_messages_request=anthropic_messages_request, model=model
)
# 5. Convert `output_format` to inline schema (Bedrock invoke doesn't support output_format)
output_format = anthropic_messages_request.pop("output_format", None)
@ -396,14 +451,14 @@ class AmazonAnthropicClaudeMessagesConfig(
output_format=output_format,
anthropic_messages_request=anthropic_messages_request,
)
# 6. AUTO-INJECT beta headers based on features used
anthropic_model_info = AnthropicModelInfo()
tools = anthropic_messages_optional_request_params.get("tools")
messages_typed = cast(List[AllMessageValues], messages)
tool_search_used = anthropic_model_info.is_tool_search_used(tools)
programmatic_tool_calling_used = anthropic_model_info.is_programmatic_tool_calling_used(
tools
programmatic_tool_calling_used = (
anthropic_model_info.is_programmatic_tool_calling_used(tools)
)
input_examples_used = anthropic_model_info.is_input_examples_used(tools)
@ -436,8 +491,7 @@ class AmazonAnthropicClaudeMessagesConfig(
if beta_set:
anthropic_messages_request["anthropic_beta"] = list(beta_set)
return anthropic_messages_request
def get_async_streaming_response_iterator(
@ -455,7 +509,7 @@ class AmazonAnthropicClaudeMessagesConfig(
)
# Convert decoded Bedrock events to Server-Sent Events expected by Anthropic clients.
return self.bedrock_sse_wrapper(
completion_stream=completion_stream,
completion_stream=completion_stream,
litellm_logging_obj=litellm_logging_obj,
request_body=request_body,
)
@ -474,14 +528,14 @@ class AmazonAnthropicClaudeMessagesConfig(
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
BaseAnthropicMessagesStreamingIterator,
)
handler = BaseAnthropicMessagesStreamingIterator(
litellm_logging_obj=litellm_logging_obj,
request_body=request_body,
)
async for chunk in handler.async_sse_wrapper(completion_stream):
yield chunk
class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder):

3
litellm/llms/databricks/chat/transformation.py
View file

@ -298,7 +298,8 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
if "reasoning_effort" in non_default_params and "claude" in model:
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
non_default_params.get("reasoning_effort")
reasoning_effort=non_default_params.get("reasoning_effort"),
model=model
)
optional_params.pop("reasoning_effort", None)
## handle thinking tokens

2
litellm/llms/deprecated_providers/palm.py
View file

@ -139,7 +139,7 @@ def completion(
)
## COMPLETION CALL
try:
response = palm.generate_text(prompt=prompt, **inference_params)
response = palm.generate_text(prompt=prompt, **inference_params) # type: ignore[attr-defined]
except Exception as e:
raise PalmError(
message=str(e),

28
litellm/llms/gigachat/chat/transformation.py
View file

@ -386,33 +386,7 @@ class GigaChatConfig(BaseConfig):
transformed.append(message)
# Collapse consecutive user messages
return self._collapse_user_messages(transformed)
def _collapse_user_messages(self, messages: List[dict]) -> List[dict]:
"""Collapse consecutive user messages into one."""
collapsed: List[dict] = []
prev_user_msg: Optional[dict] = None
content_parts: List[str] = []
for msg in messages:
if msg.get("role") == "user" and prev_user_msg is not None:
content_parts.append(msg.get("content", ""))
else:
if content_parts and prev_user_msg:
prev_user_msg["content"] = "\n".join(
[prev_user_msg.get("content", "")] + content_parts
)
content_parts = []
collapsed.append(msg)
prev_user_msg = msg if msg.get("role") == "user" else None
if content_parts and prev_user_msg:
prev_user_msg["content"] = "\n".join(
[prev_user_msg.get("content", "")] + content_parts
)
return collapsed
return transformed
def transform_response(
self,

7
litellm/llms/github_copilot/chat/transformation.py
View file

@ -1,5 +1,6 @@
from typing import List, Optional, Tuple
from litellm.exceptions import AuthenticationError
from litellm.llms.openai.openai import OpenAIConfig
from litellm.types.llms.openai import AllMessageValues
@ -29,9 +30,7 @@ class GithubCopilotConfig(OpenAIConfig):
api_key: Optional[str],
custom_llm_provider: str,
) -> Tuple[Optional[str], Optional[str], str]:
dynamic_api_base = (
self.authenticator.get_api_base() or GITHUB_COPILOT_API_BASE
)
dynamic_api_base = self.authenticator.get_api_base() or GITHUB_COPILOT_API_BASE
try:
dynamic_api_key = self.authenticator.get_api_key()
except GetAPIKeyError as e:
@ -140,7 +139,7 @@ class GithubCopilotConfig(OpenAIConfig):
"""
Check if any message contains vision content (images).
Returns True if any message has content with vision-related types, otherwise False.
Checks for:
- image_url content type (OpenAI format)
- Content items with type 'image_url'

13
litellm/llms/openai/embeddings/guardrail_translation/__init__.py Normal file
View file

@ -0,0 +1,13 @@
"""OpenAI Embeddings handler for Unified Guardrails."""
from litellm.llms.openai.embeddings.guardrail_translation.handler import (
OpenAIEmbeddingsHandler,
)
from litellm.types.utils import CallTypes
guardrail_translation_mappings = {
CallTypes.embedding: OpenAIEmbeddingsHandler,
CallTypes.aembedding: OpenAIEmbeddingsHandler,
}
__all__ = ["guardrail_translation_mappings", "OpenAIEmbeddingsHandler"]

179
litellm/llms/openai/embeddings/guardrail_translation/handler.py Normal file
View file

@ -0,0 +1,179 @@
"""
OpenAI Embeddings Handler for Unified Guardrails
This module provides guardrail translation support for OpenAI's embeddings endpoint.
The handler processes the 'input' parameter for guardrails.
"""
from typing import TYPE_CHECKING, Any, List, Optional, Union
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.utils import EmbeddingResponse
class OpenAIEmbeddingsHandler(BaseTranslation):
"""
Handler for processing OpenAI embeddings requests with guardrails.
This class provides methods to:
1. Process input text (pre-call hook)
2. Process output response (post-call hook) - embeddings don't typically need output guardrails
The handler specifically processes the 'input' parameter which can be:
- A single string
- A list of strings (for batch embeddings)
- A list of integers (token IDs - not processed by guardrails)
- A list of lists of integers (batch token IDs - not processed by guardrails)
"""
async def process_input_messages(
self,
data: dict,
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional[Any] = None,
) -> Any:
"""
Process input text by applying guardrails to text content.
Args:
data: Request data dictionary containing 'input' parameter
guardrail_to_apply: The guardrail instance to apply
litellm_logging_obj: Optional logging object
Returns:
Modified data with guardrails applied to input
"""
input_data = data.get("input")
if input_data is None:
verbose_proxy_logger.debug(
"OpenAI Embeddings: No input found in request data"
)
return data
if isinstance(input_data, str):
data = await self._process_string_input(
data, input_data, guardrail_to_apply, litellm_logging_obj
)
elif isinstance(input_data, list):
data = await self._process_list_input(
data, input_data, guardrail_to_apply, litellm_logging_obj
)
else:
verbose_proxy_logger.warning(
"OpenAI Embeddings: Unexpected input type: %s. Expected string or list.",
type(input_data),
)
return data
async def _process_string_input(
self,
data: dict,
input_data: str,
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional[Any],
) -> dict:
"""Process a single string input through the guardrail."""
inputs = GenericGuardrailAPIInputs(texts=[input_data])
if model := data.get("model"):
inputs["model"] = model
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
input_type="request",
logging_obj=litellm_logging_obj,
)
if guardrailed_texts := guardrailed_inputs.get("texts"):
data["input"] = guardrailed_texts[0]
verbose_proxy_logger.debug(
"OpenAI Embeddings: Applied guardrail to string input. "
"Original length: %d, New length: %d",
len(input_data),
len(data["input"]),
)
return data
async def _process_list_input(
self,
data: dict,
input_data: List[Union[str, int, List[int]]],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional[Any],
) -> dict:
"""Process a list input through the guardrail (if it contains strings)."""
if len(input_data) == 0:
return data
first_item = input_data[0]
# Skip non-text inputs (token IDs)
if isinstance(first_item, (int, list)):
verbose_proxy_logger.debug(
"OpenAI Embeddings: Input is token IDs, skipping guardrail processing"
)
return data
if not isinstance(first_item, str):
verbose_proxy_logger.warning(
"OpenAI Embeddings: Unexpected input list item type: %s",
type(first_item),
)
return data
# List of strings - apply guardrail
inputs = GenericGuardrailAPIInputs(texts=input_data) # type: ignore
if model := data.get("model"):
inputs["model"] = model
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
input_type="request",
logging_obj=litellm_logging_obj,
)
if guardrailed_texts := guardrailed_inputs.get("texts"):
data["input"] = guardrailed_texts
verbose_proxy_logger.debug(
"OpenAI Embeddings: Applied guardrail to %d inputs",
len(guardrailed_texts),
)
return data
async def process_output_response(
self,
response: "EmbeddingResponse",
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional[Any] = None,
user_api_key_dict: Optional[Any] = None,
) -> Any:
"""
Process output response - embeddings responses contain vectors, not text.
For embeddings, the output is numerical vectors, so there's typically
no text content to apply guardrails to. This method is a no-op but
is included for interface consistency.
Args:
response: Embedding response object
guardrail_to_apply: The guardrail instance to apply
litellm_logging_obj: Optional logging object
user_api_key_dict: User API key metadata
Returns:
Unmodified response (embeddings don't have text output to guard)
"""
verbose_proxy_logger.debug(
"OpenAI Embeddings: Output response processing skipped - "
"embeddings contain vectors, not text"
)
return response

2
litellm/llms/vertex_ai/files/transformation.py
View file

@ -165,7 +165,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
"""
Get the complete url for the request
"""
bucket_name = litellm_params.get("bucket_name") or os.getenv("GCS_BUCKET_NAME")
bucket_name = litellm_params.get("bucket_name") or litellm_params.get("litellm_metadata", {}).pop("gcs_bucket_name", None) or os.getenv("GCS_BUCKET_NAME")
if not bucket_name:
raise ValueError("GCS bucket_name is required")
file_data = data.get("file")

55
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -1732,6 +1732,52 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
else:
return "stop"
@staticmethod
def _check_prompt_level_content_filter(
processed_chunk: GenerateContentResponseBody,
response_id: Optional[str],
) -> Optional["ModelResponseStream"]:
"""
Check if prompt is blocked due to content filtering at the prompt level.
This handles the case where Vertex AI blocks the prompt before generation begins,
indicated by promptFeedback.blockReason being present.
Args:
processed_chunk: The parsed response chunk from Vertex AI
response_id: The response ID from the chunk
Returns:
ModelResponseStream with content_filter finish_reason if blocked, None otherwise.
Note:
This is consistent with non-streaming _handle_blocked_response() behavior.
Candidate-level content filtering (SAFETY, RECITATION, etc.) is handled
separately via _process_candidates() → _check_finish_reason().
"""
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
# Check if prompt is blocked due to content filtering
prompt_feedback = processed_chunk.get("promptFeedback")
if prompt_feedback and "blockReason" in prompt_feedback:
verbose_logger.debug(
f"Prompt blocked due to: {prompt_feedback.get('blockReason')} - {prompt_feedback.get('blockReasonMessage')}"
)
# Create a content_filter response (consistent with non-streaming _handle_blocked_response)
choice = StreamingChoices(
finish_reason="content_filter",
index=0,
delta=Delta(content=None, role="assistant"),
logprobs=None,
enhancements=None,
)
model_response = ModelResponseStream(choices=[choice], id=response_id)
return model_response
return None
@staticmethod
def _calculate_web_search_requests(grounding_metadata: List[dict]) -> Optional[int]:
web_search_requests: Optional[int] = None
@ -2813,6 +2859,15 @@ class ModelResponseIterator:
processed_chunk = GenerateContentResponseBody(**chunk) # type: ignore
response_id = processed_chunk.get("responseId")
model_response = ModelResponseStream(choices=[], id=response_id)
# Check if prompt is blocked due to content filtering
blocked_response = VertexGeminiConfig._check_prompt_level_content_filter(
processed_chunk=processed_chunk,
response_id=response_id,
)
if blocked_response is not None:
model_response = blocked_response
usage: Optional[Usage] = None
_candidates: Optional[List[Candidates]] = processed_chunk.get("candidates")
grounding_metadata: List[dict] = []

34
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py
View file

@ -1,5 +1,8 @@
from typing import Any, Dict, List, Optional, Tuple
from litellm.anthropic_beta_headers_manager import (
update_headers_with_filtered_beta,
)
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
@ -7,7 +10,6 @@ from litellm.llms.anthropic.experimental_pass_through.messages.transformation im
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_PROMPT_CACHING_SCOPE_BETA_HEADER,
)
from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
from litellm.types.llms.vertex_ai import VertexPartnerProvider
@ -65,10 +67,6 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
existing_beta = headers.get("anthropic-beta")
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
# Use the helper to remove unsupported beta headers
self.remove_unsupported_beta(headers)
beta_values.discard(ANTHROPIC_PROMPT_CACHING_SCOPE_BETA_HEADER)
# Check for web search tool
for tool in tools:
@ -84,6 +82,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
if beta_values:
headers["anthropic-beta"] = ",".join(beta_values)
# Filter out unsupported beta headers for Vertex AI
headers = update_headers_with_filtered_beta(
headers=headers,
provider="vertex_ai",
)
return headers, api_base
def get_complete_url(
@ -128,23 +132,3 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
) # do not pass output_format in request body to vertex ai - vertex ai does not support output_format as yet
return anthropic_messages_request
def remove_unsupported_beta(self, headers: dict) -> None:
"""
Helper method to remove unsupported beta headers from the beta headers.
Modifies headers in place.
"""
unsupported_beta_headers = [
ANTHROPIC_PROMPT_CACHING_SCOPE_BETA_HEADER
]
existing_beta = headers.get("anthropic-beta")
if existing_beta:
filtered_beta = [
b.strip()
for b in existing_beta.split(",")
if b.strip() not in unsupported_beta_headers
]
if filtered_beta:
headers["anthropic-beta"] = ",".join(filtered_beta)
elif "anthropic-beta" in headers:
del headers["anthropic-beta"]

39
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
View file

@ -51,6 +51,40 @@ class VertexAIAnthropicConfig(AnthropicConfig):
def custom_llm_provider(self) -> Optional[str]:
return "vertex_ai"
def _add_context_management_beta_headers(
self, beta_set: set, context_management: dict
) -> None:
"""
Add context_management beta headers to the beta_set.
- If any edit has type "compact_20260112", add compact-2026-01-12 header
- For all other edits, add context-management-2025-06-27 header
Args:
beta_set: Set of beta headers to modify in-place
context_management: The context_management dict from optional_params
"""
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
edits = context_management.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
def transform_request(
self,
model: str,
@ -86,6 +120,11 @@ class VertexAIAnthropicConfig(AnthropicConfig):
beta_set = set(auto_betas)
if tool_search_used:
beta_set.add("tool-search-tool-2025-10-19") # Vertex requires this header for tool search
# Add context_management beta headers (compact and/or context-management)
context_management = optional_params.get("context_management")
if context_management:
self._add_context_management_beta_headers(beta_set, context_management)
if beta_set:
data["anthropic_beta"] = list(beta_set)

477
litellm/model_prices_and_context_window_backup.json
View file

@ -963,6 +963,276 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"global.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"global.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"us.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.75e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"eu.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.75e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"eu.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.75e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"apac.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.75e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"apac.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.75e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"anthropic.claude-sonnet-4-20250514-v1:0": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
@ -1444,6 +1714,33 @@
"supports_tool_choice": true,
"supports_vision": true
},
"azure_ai/claude-opus-4-6": {
"input_cost_per_token": 5e-06,
"output_cost_per_token": 2.5e-05,
"litellm_provider": "azure_ai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"azure_ai/claude-opus-4-1": {
"cache_creation_input_token_cost": 1.875e-05,
"cache_creation_input_token_cost_above_1hr": 3e-05,
@ -7455,6 +7752,130 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"claude-opus-4-6": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"us/claude-opus-4-6": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.75e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"us/claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.75e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"claude-sonnet-4-20250514": {
"deprecation_date": "2026-05-14",
"cache_creation_input_token_cost": 3.75e-06,
@ -10575,6 +10996,32 @@
"/v1/audio/transcriptions"
]
},
"elevenlabs/eleven_v3": {
"input_cost_per_character": 0.00018,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_multilingual_v2": {
"input_cost_per_character": 0.00018,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"embed-english-light-v2.0": {
"input_cost_per_token": 1e-07,
"litellm_provider": "cohere",
@ -29291,6 +29738,36 @@
"tool_use_system_prompt_tokens": 159,
"supports_native_streaming": true
},
"vertex_ai/claude-opus-4-6": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"vertex_ai/claude-sonnet-4-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,

2
litellm/proxy/_experimental/mcp_server/auth/litellm_auth_handler.py
View file

@ -27,6 +27,7 @@ class MCPAuthenticatedUser(AuthenticatedUser):
oauth2_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
raw_headers: Optional[Dict[str, str]] = None,
client_ip: Optional[str] = None,
):
self.user_api_key_auth = user_api_key_auth
self.mcp_auth_header = mcp_auth_header
@ -35,3 +36,4 @@ class MCPAuthenticatedUser(AuthenticatedUser):
self.mcp_protocol_version = mcp_protocol_version
self.oauth2_headers = oauth2_headers
self.raw_headers = raw_headers
self.client_ip = client_ip

52
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -1,11 +1,12 @@
from typing import Dict, List, Optional, Set, Tuple
from fastapi import HTTPException
from starlette.datastructures import Headers
from starlette.requests import Request
from starlette.types import Scope
from litellm._logging import verbose_logger
from litellm.proxy._types import LiteLLM_TeamTable, SpecialHeaders, UserAPIKeyAuth
from litellm.proxy._types import LiteLLM_TeamTable, ProxyException, SpecialHeaders, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@ -63,6 +64,13 @@ class MCPRequestHandler:
HTTPException: If headers are invalid or missing required headers
"""
headers = MCPRequestHandler._safe_get_headers_from_scope(scope)
# Check if there is an explicit LiteLLM API key (primary header)
has_explicit_litellm_key = (
headers.get(MCPRequestHandler.LITELLM_API_KEY_HEADER_NAME_PRIMARY)
is not None
)
litellm_api_key = (
MCPRequestHandler.get_litellm_api_key_from_headers(headers) or ""
)
@ -106,16 +114,38 @@ class MCPRequestHandler:
request.body = mock_body # type: ignore
if ".well-known" in str(request.url): # public routes
validated_user_api_key_auth = UserAPIKeyAuth()
# elif litellm_api_key == "":
# from fastapi import HTTPException
# raise HTTPException(
# status_code=401,
# detail="LiteLLM API key is missing. Please add it or use OAuth authentication.",
# headers={
# "WWW-Authenticate": f'Bearer resource_metadata=f"{request.base_url}/.well-known/oauth-protected-resource"',
# },
# )
elif has_explicit_litellm_key:
# Explicit x-litellm-api-key provided - always validate normally
validated_user_api_key_auth = await user_api_key_auth(
api_key=litellm_api_key, request=request
)
elif oauth2_headers:
# No x-litellm-api-key, but Authorization header present.
# Could be a LiteLLM key (backward compat) OR an OAuth2 token
# from an upstream MCP provider (e.g. Atlassian).
# Try LiteLLM auth first; on auth failure, treat as OAuth2 passthrough.
try:
validated_user_api_key_auth = await user_api_key_auth(
api_key=litellm_api_key, request=request
)
except HTTPException as e:
if e.status_code in (401, 403):
verbose_logger.debug(
"MCP OAuth2: Authorization header is not a valid LiteLLM key, "
"treating as OAuth2 token passthrough"
)
validated_user_api_key_auth = UserAPIKeyAuth()
else:
raise
except ProxyException as e:
if str(e.code) in ("401", "403"):
verbose_logger.debug(
"MCP OAuth2: Authorization header is not a valid LiteLLM key, "
"treating as OAuth2 token passthrough"
)
validated_user_api_key_auth = UserAPIKeyAuth()
else:
raise
else:
validated_user_api_key_auth = await user_api_key_auth(
api_key=litellm_api_key, request=request

28
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -9,13 +9,14 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
from litellm.types.mcp_server.mcp_server_manager import MCPServer
from litellm.proxy.utils import get_server_root_path
from litellm.types.mcp_server.mcp_server_manager import MCPServer
router = APIRouter(
tags=["mcp"],
@ -300,7 +301,10 @@ async def authorize(
)
lookup_name = mcp_server_name or client_id
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(lookup_name)
client_ip = IPAddressUtils.get_mcp_client_ip(request)
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
lookup_name, client_ip=client_ip
)
if mcp_server is None:
raise HTTPException(status_code=404, detail="MCP server not found")
return await authorize_with_server(
@ -342,7 +346,10 @@ async def token_endpoint(
)
lookup_name = mcp_server_name or client_id
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(lookup_name)
client_ip = IPAddressUtils.get_mcp_client_ip(request)
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
lookup_name, client_ip=client_ip
)
if mcp_server is None:
raise HTTPException(status_code=404, detail="MCP server not found")
return await exchange_token_with_server(
@ -425,7 +432,10 @@ def _build_oauth_protected_resource_response(
request_base_url = get_request_base_url(request)
mcp_server: Optional[MCPServer] = None
if mcp_server_name:
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name)
client_ip = IPAddressUtils.get_mcp_client_ip(request)
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
mcp_server_name, client_ip=client_ip
)
# Build resource URL based on the pattern
if mcp_server_name:
@ -538,7 +548,10 @@ def _build_oauth_authorization_server_response(
mcp_server: Optional[MCPServer] = None
if mcp_server_name:
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name)
client_ip = IPAddressUtils.get_mcp_client_ip(request)
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
mcp_server_name, client_ip=client_ip
)
return {
"issuer": request_base_url, # point to your proxy
@ -629,7 +642,10 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non
if not mcp_server_name:
return dummy_return
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name)
client_ip = IPAddressUtils.get_mcp_client_ip(request)
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
mcp_server_name, client_ip=client_ip
)
if mcp_server is None:
return dummy_return
return await register_client_with_server(

Some files were not shown because too many files have changed in this diff Show more