scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge branch 'main' into litellm_fix_tts_metrics

Browse source
This commit is contained in:
Harshit Jain 2026-02-12 09:33:55 +05:30 • committed by GitHub
commit af8dabb4ce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1152 changed files with 54028 additions and 11399 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
.circleci/config.yml
View file

@ -112,6 +112,24 @@ jobs:
python -m mypy .
cd ..
no_output_timeout: 10m
semgrep:
docker:
- image: cimg/python:3.12
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Semgrep
command: pip install semgrep
- run:
name: Run Semgrep (custom rules only)
command: semgrep scan --config .semgrep/rules . --error
local_testing_part1:
docker:
- image: cimg/python:3.12
@ -1372,6 +1390,51 @@ jobs:
paths:
- mcp_coverage.xml
- mcp_coverage
agent_testing:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.11.0"
pip install "a2a-sdk"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml agent_coverage.xml
mv .coverage agent_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- agent_coverage.xml
- agent_coverage
guardrails_testing:
docker:
- image: cimg/python:3.11
@ -2232,6 +2295,7 @@ jobs:
- run: python ./tests/code_coverage_tests/router_code_coverage.py
- run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
- run: python ./tests/code_coverage_tests/info_log_check.py
- run: python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
- run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
- run: python ./tests/code_coverage_tests/code_qa_check_tests.py
- run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
@ -3756,7 +3820,6 @@ jobs:
- run:
name: Get new version
command: |
cd litellm-proxy-extras
NEW_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV
@ -3781,7 +3844,6 @@ jobs:
- run:
name: Publish to PyPI
command: |
cd litellm-proxy-extras
echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
python -m pip install --upgrade pip build twine setuptools wheel
rm -rf build dist
@ -3888,6 +3950,9 @@ jobs:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
parameters:
browser:
type: string
steps:
- checkout
- setup_google_dns
@ -3917,7 +3982,7 @@ jobs:
echo "Expires at: $EXPIRES_AT"
neon branches create \
--project-id $NEON_PROJECT_ID \
--name preview/commit-${CIRCLE_SHA1:0:7} \
--name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--expires-at $EXPIRES_AT \
--parent br-fancy-paper-ad1olsb3 \
--api-key $NEON_API_KEY || true
@ -3927,7 +3992,7 @@ jobs:
E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch preview/commit-${CIRCLE_SHA1:0:7} \
--branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--database-name yuneng-trial-db \
--role neondb_owner)
echo $E2E_UI_TEST_DATABASE_URL
@ -3939,7 +4004,7 @@ jobs:
-e UI_USERNAME="admin" \
-e UI_PASSWORD="gm" \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
--name litellm-docker-database \
--name litellm-docker-database-<< parameters.browser >> \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
@ -3955,7 +4020,7 @@ jobs:
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f litellm-docker-database
command: docker logs -f litellm-docker-database-<< parameters.browser >>
background: true
- run:
name: Wait for app to be ready
@ -3964,6 +4029,7 @@ jobs:
name: Run Playwright Tests
command: |
npx playwright test \
--project << parameters.browser >> \
--config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
--reporter=html \
--output=test-results
@ -4070,6 +4136,12 @@ workflows:
only:
- main
- /litellm_.*/
- semgrep:
filters:
branches:
only:
- main
- /litellm_.*/
- local_testing_part1:
filters:
branches:
@ -4169,6 +4241,20 @@ workflows:
- main
- /litellm_.*/
- e2e_ui_testing:
name: e2e_ui_testing_chromium
browser: chromium
context: e2e_ui_tests
requires:
- ui_build
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- e2e_ui_testing:
name: e2e_ui_testing_firefox
browser: firefox
context: e2e_ui_tests
requires:
- ui_build
@ -4264,6 +4350,12 @@ workflows:
only:
- main
- /litellm_.*/
- agent_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- guardrails_testing:
filters:
branches:
@ -4371,6 +4463,7 @@ workflows:
- llm_translation_testing
- realtime_translation_testing
- mcp_testing
- agent_testing
- google_generate_content_endpoint_testing
- guardrails_testing
- llm_responses_api_testing
@ -4441,6 +4534,7 @@ workflows:
- publish_to_pypi:
requires:
- mypy_linting
- semgrep
- local_testing_part1
- local_testing_part2
- build_and_test
@ -4449,6 +4543,7 @@ workflows:
- llm_translation_testing
- realtime_translation_testing
- mcp_testing
- agent_testing
- google_generate_content_endpoint_testing
- llm_responses_api_testing
- ocr_testing
@ -4472,7 +4567,8 @@ workflows:
- litellm_assistants_api_testing
- auth_ui_unit_tests
- db_migration_disable_update_check
- e2e_ui_testing
- e2e_ui_testing_chromium
- e2e_ui_testing_firefox
- litellm_proxy_unit_testing_key_generation
- litellm_proxy_unit_testing_part1
- litellm_proxy_unit_testing_part2

2
.dockerignore
View file

@ -48,7 +48,7 @@ dist/
build/
*.egg-info/
.DS_Store
node_modules/
**/node_modules
*.log
.env
.env.local

1
.github/pull_request_template.md vendored
View file

@ -9,6 +9,7 @@
- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
## CI (LiteLLM team)

52
.semgrep/rules/README.md Normal file
View file

@ -0,0 +1,52 @@
# Custom Semgrep Rules
All `.yml` files under `.semgrep/rules/` run in CI (CircleCI `semgrep` job).
## Add a Rule
* Add a `.yml` file under `.semgrep/rules/<language>/<domain>/`
[Rule syntax →](https://semgrep.dev/docs/writing-rules/rule-syntax/)
## Organizing Rules
### Structure: language → domain
```
.semgrep/rules/<language>/<domain>/<rule-name>.yml
```
Examples:
- `python/security/unsafe-yaml-load.yml`
- `python/reliability/missing-timeout-http.yml`
- `python/performance/blocking-io-in-async.yml`
### Rule metadata
Match tags to the folder for consistent filtering:
```yaml
metadata:
tags: [python, security]
```
### Severity expectations
All rules must fail CI on findings. No warn-only rules.
- Use `severity: ERROR` in rule metadata
- If a rule is noisy → refine until low false positives before adding
## Run Locally
```bash
semgrep scan --config .semgrep/rules . --error
```
With Semgrep registry:
```bash
semgrep scan --config auto --config .semgrep/rules .
```

17
.semgrep/rules/python/reliability/unbounded-memory.yml Normal file
View file

@ -0,0 +1,17 @@
# Unbounded memory growth – data structures without a clear max limit
# Can lead to OOM under load.
rules:
- id: unbounded-asyncio-queue
message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues).
severity: ERROR
languages: [python]
pattern-either:
- pattern: asyncio.Queue()
- pattern: asyncio.Queue(maxsize=0)
metadata:
category: reliability
cwe: "CWE-400: Uncontrolled Resource Consumption"
tags: [python, reliability]
confidence: HIGH
source: https://docs.python.org/3/library/asyncio-queue.html

1
CLAUDE.md
View file

@ -90,6 +90,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- Pydantic v2 for data validation
- Async/await patterns throughout
- Type hints required for all public APIs
- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
### Testing Strategy
- Unit tests in `tests/test_litellm/`

48
CONTRIBUTING.md
View file

@ -7,11 +7,20 @@ Thank you for your interest in contributing to LiteLLM! We welcome contributions
Here are the core requirements for any PR submitted to LiteLLM:
- [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
#### Proxy (Backend) PRs
- [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
- [ ] **Ensure your PR passes all checks**:
- [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
- [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
#### UI PRs
- [ ] **Ensure the UI builds successfully** - `npm run build`
- [ ] **Ensure all UI unit tests pass** - `npm run test`
- [ ] **Add tests for new components or logic** - If you are adding a new component or new logic, add corresponding tests
## **Contributor License Agreement (CLA)**
@ -245,6 +254,43 @@ docker run \
--config /app/config.yaml --detailed_debug
```
## UI Development
### 1. Setup Your Local UI Development Environment
```bash
# Clone the repo (if you haven't already)
git clone https://github.com/YOUR_USERNAME/litellm.git
cd litellm
# Navigate to the UI dashboard directory
cd ui/litellm-dashboard
# Install dependencies
npm install
# Start the development server
npm run dev
```
### 2. Adding UI Tests
If you are adding a **new component** or **new logic**, you must add corresponding tests.
### 3. Running UI Unit Tests
```bash
npm run test
```
### 4. Building the UI
Ensure the UI builds successfully before submitting your PR:
```bash
npm run build
```
## Submitting Your PR
1. **Push your branch**: `git push origin your-feature-branch`

36
Dockerfile
View file

@ -3,6 +3,7 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
@ -48,7 +49,22 @@ USER root
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@latest
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
# SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
# levels inside its dependency tree. `npm install -g <pkg>` only creates a
# SEPARATE global package, it does NOT replace npm's internal copies.
# We must find and replace EVERY copy inside npm's directory.
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
npm cache clean --force
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -62,10 +78,28 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi
# Remove test files and keys from dependencies
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done
# Install semantic_router and aurelio-sdk using script
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh

2
README.md
View file

@ -309,7 +309,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
| [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ | | | ✅ | | | | |
| [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | | | | | | | |
| [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | | | | | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | | ✅ | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |

33
ci_cd/.grype.yaml
View file

@ -1,3 +1,36 @@
ignore:
- vulnerability: CVE-2026-22184
reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
# Wolfi base image: Python 3.13 and Node from apk have no fixed builds in Wolfi yet / not applicable
- vulnerability: CVE-2025-55130
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-59465
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-55131
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-59466
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2026-21637
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-55132
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: GHSA-hx9q-6w63-j58v
reason: orjson dumps recursion; allowlisted
- vulnerability: GHSA-73rr-hh4g-fpgx
reason: diff npm transitive dep; override in package.json, allowlisted
- vulnerability: CVE-2026-0865
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15282
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2026-0672
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15366
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15367
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-11468
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-12781
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2026-1299
reason: Python 3.13 in Wolfi base; no fixed apk build yet

15
ci_cd/security_scans.sh
View file

@ -140,12 +140,14 @@ run_grype_scans() {
"GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
"GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
"CVE-2025-59465" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-55131" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59466" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-55130" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59467" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2026-21637" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59465" # Node only used for Admin UI build/prisma
"CVE-2025-55131" # Node only used for Admin UI build/prisma
"CVE-2025-59466" # Node only used for Admin UI build/prisma
"CVE-2025-55130" # Node only used for Admin UI build/prisma
"CVE-2025-59467" # Node only used for Admin UI build/prisma
"CVE-2026-21637" # Node only used for Admin UI build/prisma
"CVE-2025-55132" # Node only used for Admin UI build/prisma
"GHSA-hx9q-6w63-j58v" # orjson dumps recursion; allowlisted
"CVE-2025-15281" # No fix available yet
"CVE-2026-0865" # No fix available yet
"CVE-2025-15282" # No fix available yet
@ -155,6 +157,7 @@ run_grype_scans() {
"CVE-2025-12781" # No fix available yet
"CVE-2025-11468" # No fix available yet
"CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
"CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
)
# Build JSON array of allowlisted CVE IDs for jq

6
cookbook/nova_sonic_realtime.py
View file

@ -16,10 +16,14 @@ Usage:
import asyncio
import base64
import json
import os
import pyaudio
import websockets
from typing import Optional
# Bounded queue size for audio chunks (configurable via env to avoid unbounded memory)
AUDIO_QUEUE_MAXSIZE = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 10_000))
# Audio configuration (matching Nova Sonic requirements)
INPUT_SAMPLE_RATE = 16000 # Nova Sonic expects 16kHz input
OUTPUT_SAMPLE_RATE = 24000 # Nova Sonic outputs 24kHz
@ -40,7 +44,7 @@ class RealtimeClient:
self.api_key = api_key
self.ws: Optional[websockets.WebSocketClientProtocol] = None
self.is_active = False
self.audio_queue = asyncio.Queue()
self.audio_queue = asyncio.Queue(maxsize=AUDIO_QUEUE_MAXSIZE)
self.pyaudio = pyaudio.PyAudio()
self.input_stream = None
self.output_stream = None

13
docker/Dockerfile.custom_ui
View file

@ -6,7 +6,18 @@ WORKDIR /app
# Install Node.js and npm (adjust version as needed)
RUN apt-get update && apt-get install -y nodejs npm && \
npm install -g npm@latest tar@latest
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
npm cache clean --force
# Copy the UI source into the container
COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard

27
docker/Dockerfile.database
View file

@ -50,7 +50,18 @@ USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@latest
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
npm cache clean --force
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -64,6 +75,20 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done
# Install semantic_router and aurelio-sdk using script
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh

27
docker/Dockerfile.dev
View file

@ -62,7 +62,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@latest tar@latest
&& npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done \
&& npm cache clean --force
WORKDIR /app
@ -80,6 +91,20 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
rm -f *.whl && \
rm -rf /wheels
# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done
# Generate prisma client and set permissions
# Convert Windows line endings to Unix for entrypoint scripts
RUN prisma generate && \

27
docker/Dockerfile.non_root
View file

@ -104,7 +104,18 @@ RUN for i in 1 2 3; do \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done \
&& npm install -g npm@latest tar@latest
&& npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done \
&& npm cache clean --force
# Copy artifacts from builder
COPY --from=builder /app/requirements.txt /app/requirements.txt
@ -146,6 +157,20 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
fi; \
fi
# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done
# Permissions, cleanup, and Prisma prep
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && \

491
docs/my-website/blog/claude_opus_4_6/index.md
View file

@ -3,6 +3,10 @@ slug: claude_opus_4_6
title: "Day 0 Support: Claude Opus 4.6"
date: 2026-02-05T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
@ -219,6 +223,489 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
</TabItem>
</Tabs>
## More Features Coming Soon
## Advanced Features
We're actively working on supporting new features for Claude Opus 4.6. Stay tuned for updates!
### Compaction
<Tabs>
<TabItem value="completions" label="/chat/completions">
Litellm supports enabling compaction for the new claude-opus-4-6.
**Enabling Compaction**
To enable compaction, add the `context_management` parameter with the `compact_20260112` edit type:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "What is the weather in San Francisco?"
}
],
"context_management": {
"edits": [
{
"type": "compact_20260112"
}
]
},
"max_tokens": 100
}'
```
All the parameters supported for context_management by anthropic are supported and can be directly added. Litellm automatically adds the `compact-2026-01-12` beta header in the request.
</TabItem>
<TabItem value="messages" label="/v1/messages">
Enable compaction to reduce context size while preserving key information. LiteLLM automatically adds the `compact-2026-01-12` beta header when compaction is enabled.
:::info
**Provider Support:** Compaction is supported on Anthropic, Azure AI, and Vertex AI. It is **not supported** on Bedrock (Invoke or Converse APIs).
:::
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 4096,
"messages": [
{
"role": "user",
"content": "Hi"
}
],
"context_management": {
"edits": [
{
"type": "compact_20260112"
}
]
}
}'
```
</TabItem>
</Tabs>
**Response with Compaction Block**
The response will include the compaction summary in `provider_specific_fields.compaction_blocks`:
```json
{
"id": "chatcmpl-a6c105a3-4b25-419e-9551-c800633b6cb2",
"created": 1770357619,
"model": "claude-opus-4-6",
"object": "chat.completion",
"choices": [
{
"finish_reason": "length",
"index": 0,
"message": {
"content": "I don't have access to real-time data, so I can't provide the current weather in San Francisco. To get up-to-date weather information, I'd recommend checking:\n\n- **Weather websites** like weather.com, accuweather.com, or wunderground.com\n- **Search engines** – just Google \"San Francisco weather\"\n- **Weather apps** on your phone (e.g., Apple Weather, Google Weather)\n- **National",
"role": "assistant",
"provider_specific_fields": {
"compaction_blocks": [
{
"type": "compaction",
"content": "Summary of the conversation: The user requested help building a web scraper..."
}
]
}
}
}
],
"usage": {
"completion_tokens": 100,
"prompt_tokens": 86,
"total_tokens": 186
}
}
```
**Using Compaction Blocks in Follow-up Requests**
To continue the conversation with compaction, include the compaction block in the assistant message's `provider_specific_fields`:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "How can I build a web scraper?"
},
{
"role": "assistant",
"content": [
{
"type": "text",
"text": "Certainly! To build a basic web scraper, you'll typically use a programming language like Python along with libraries such as `requests` (for fetching web pages) and `BeautifulSoup` (for parsing HTML). Here's a basic example:\n\n```python\nimport requests\nfrom bs4 import BeautifulSoup\n\nurl = 'https://example.com'\nresponse = requests.get(url)\nsoup = BeautifulSoup(response.text, 'html.parser')\n\n# Extract and print all text\ntext = soup.get_text()\nprint(text)\n```\n\nLet me know what you're interested in scraping or if you need help with a specific website!"
}
],
"provider_specific_fields": {
"compaction_blocks": [
{
"type": "compaction",
"content": "Summary of the conversation: The user asked how to build a web scraper, and the assistant gave an overview using Python with requests and BeautifulSoup."
}
]
}
},
{
"role": "user",
"content": "How do I use it to scrape product prices?"
}
],
"context_management": {
"edits": [
{
"type": "compact_20260112"
}
]
},
"max_tokens": 100
}'
```
**Streaming Support**
Compaction blocks are also supported in streaming mode. You'll receive:
- `compaction_start` event when a compaction block begins
- `compaction_delta` events with the compaction content
- The accumulated `compaction_blocks` in `provider_specific_fields`
### Adaptive Thinking
<Tabs>
<TabItem value="completions" label="/chat/completions">
LiteLLM supports adaptive thinking through the `reasoning_effort` parameter:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "Solve this complex problem: What is the optimal strategy for..."
}
],
"reasoning_effort": "high"
}'
```
</TabItem>
<TabItem value="messages" label="/v1/messages">
Use the `thinking` parameter with `type: "adaptive"` to enable adaptive thinking mode:
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 16000,
"thinking": {
"type": "adaptive"
},
"messages": [
{
"role": "user",
"content": "Explain why the sum of two even numbers is always even."
}
]
}'
```
</TabItem>
</Tabs>
### Effort Levels
<Tabs>
<TabItem value="completions" label="/chat/completions">
Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "Explain quantum computing"
}
],
"output_config": {
"effort": "medium"
}
}'
```
You can use reasoning effort plus output_config to have more control on the model.
</TabItem>
<TabItem value="messages" label="/v1/messages">
Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter:
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 4096,
"messages": [
{
"role": "user",
"content": "Explain quantum computing"
}
],
"output_config": {
"effort": "medium"
}
}'
```
</TabItem>
</Tabs>
### 1M Token Context (Beta)
Opus 4.6 supports 1M token context. Premium pricing applies for prompts exceeding 200k tokens ($10/$37.50 per million input/output tokens). LiteLLM supports cost calculations for 1M token contexts.
<Tabs>
<TabItem value="completions" label="/chat/completions">
To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider.
**Step 1: Enable header forwarding in your config**
```yaml
general_settings:
forward_client_headers_to_llm_api: true
```
**Step 2: Send requests with the beta header**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--header 'anthropic-beta: context-1m-2025-08-07' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "Analyze this large document..."
}
]
}'
```
</TabItem>
<TabItem value="messages" label="/v1/messages">
To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider.
**Step 1: Enable header forwarding in your config**
```yaml
general_settings:
forward_client_headers_to_llm_api: true
```
**Step 2: Send requests with the beta header**
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'anthropic-beta: context-1m-2025-08-07' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 16000,
"messages": [
{
"role": "user",
"content": "Analyze this large document..."
}
]
}'
```
:::tip
You can combine multiple beta headers by separating them with commas:
```bash
--header 'anthropic-beta: context-1m-2025-08-07,compact-2026-01-12'
```
:::
</TabItem>
</Tabs>
### US-Only Inference
Available at 1.1× token pricing. LiteLLM automatically tracks costs for US-only inference.
<Tabs>
<TabItem value="completions" label="/chat/completions">
Use the `inference_geo` parameter to specify US-only inference:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
],
"inference_geo": "us"
}'
```
LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking.
</TabItem>
<TabItem value="messages" label="/v1/messages">
Use the `inference_geo` parameter to specify US-only inference:
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 4096,
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
],
"inference_geo": "us"
}'
```
LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking.
</TabItem>
</Tabs>
### Fast Mode
:::info
Fast mode is **only supported on the Anthropic provider** (`anthropic/claude-opus-4-6`). It is not available on Azure AI, Vertex AI, or Bedrock.
:::
**Pricing:**
- Standard: $5 input / $25 output per MTok
- Fast: $30 input / $150 output per MTok (6× premium)
<Tabs>
<TabItem value="completions" label="/chat/completions">
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "Refactor this module..."
}
],
"max_tokens": 4096,
"speed": "fast"
}'
```
**Using OpenAI SDK:**
```python
import openai
client = openai.OpenAI(
api_key="your-litellm-key",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="claude-opus-4-6",
messages=[{"role": "user", "content": "Refactor this module..."}],
max_tokens=4096,
extra_body={"speed": "fast"}
)
```
**Using LiteLLM SDK:**
```python
from litellm import completion
response = completion(
model="anthropic/claude-opus-4-6",
messages=[{"role": "user", "content": "Refactor this module..."}],
max_tokens=4096,
speed="fast"
)
```
LiteLLM automatically tracks the higher costs for fast mode in usage and cost calculations.
</TabItem>
<TabItem value="messages" label="/v1/messages">
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 4096,
"speed": "fast",
"messages": [
{
"role": "user",
"content": "Refactor this module..."
}
]
}'
```
LiteLLM automatically:
- Adds the `fast-mode-2026-02-01` beta header
- Tracks the 6× premium pricing in cost calculations
</TabItem>
</Tabs>

220
docs/my-website/blog/fastapi_middleware_performance/index.mdx Normal file
View file

@ -0,0 +1,220 @@
---
slug: fastapi-middleware-performance
title: "Your Middleware Could Be a Bottleneck"
date: 2026-02-07T10:00:00
authors:
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Ryan Crabbe
title: "Performance Engineer, LiteLLM"
url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
description: "How we improved LiteLLM proxy latency and throughput by replacing a single middleware base class"
tags: [performance, fastapi, middleware]
hide_table_of_contents: false
---
import { BaseHTTPMiddlewareAnimation, PureASGIAnimation, BenchmarkVisualization } from '@site/src/components/MiddlewareDiagrams';
> How we improved LiteLLM proxy latency and throughput by replacing a single, simple middleware base class
---
## Our Setup
The LiteLLM proxy server has two middleware layers. The first is Starlette's `CORSMiddleware` (re-exported by FastAPI), which is a pure ASGI middleware. Then we have a simple BaseHTTPMiddleware called PrometheusAuthMiddleware.
The job of `PrometheusAuthMiddleware` is to authenticate requests to the `/metrics` endpoint. It's not on by default, you enable it with a flag in your proxy config:
<details>
<summary>Proxy config flag</summary>
```yaml
litellm_settings:
require_auth_for_metrics_endpoint: true
```
</details>
The middleware checks two things: is the request hitting `/metrics`, and is auth even enabled? If both checks fail, which they do for the vast majority of requests, it just passes the request through unchanged.
<details>
<summary>PrometheusAuthMiddleware source</summary>
```python
class PrometheusAuthMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
if self._is_prometheus_metrics_endpoint(request):
if self._should_run_auth_on_metrics_endpoint() is True:
try:
await user_api_key_auth(request=request, api_key=...)
except Exception as e:
return JSONResponse(status_code=401, content=...)
response = await call_next(request)
return response
@staticmethod
def _is_prometheus_metrics_endpoint(request: Request):
if "/metrics" in request.url.path:
return True
return False
```
</details>
Looks harmless. Subclass `BaseHTTPMiddleware`, implement `dispatch()`, done. This is what you will see in Starlette's documentation<sup>[1](#footnote-1)</sup>.
{/* truncate */}
---
## What BaseHTTPMiddleware Actually Does
When you write a `dispatch()` method, you'd expect the request to flow straight through your function and out the other side. What actually happens is much more involved.
On every request, even a pure passthrough (meaning nothing happens), `BaseHTTPMiddleware` creates **7 intermediate objects and tasks**:
<BaseHTTPMiddlewareAnimation />
It wraps the request in a new object to track body state, creates a synchronization event, allocates an in-memory channel to pass messages between your middleware and the inner app, sets up a task group to manage the lifecycle, and then runs your actual route handler in a *separate background task* when you call `call_next()`. The response body then flows back through that in-memory channel, gets re-wrapped in a streaming response object, and finally reaches the caller. That's a lot.
For a middleware that for us, does nothing on 99.9% of requests, paying this cost doesn't make sense.
Compare that to a pure ASGI middleware, which we can have just check the request path and continue along.
<PureASGIAnimation />
Our middleware is doing something really simple. For the vast majority of requests it doesn't need to do anything at all but just let the request pass through. It doesn't need task groups, memory streams, or cancel scopes. It needs a function call.
---
## Comparing Both
We replaced the `BaseHTTPMiddleware` subclass with a pure ASGI middleware. To benchmark the difference, we used Apache Bench<sup>[2](#footnote-2)</sup> to compare both configurations of LiteLLM's middleware stack: the old setup (1 pure ASGI + 1 `BaseHTTPMiddleware`) against the new setup (2 pure ASGI).
A minimal FastAPI app serves `GET /health` → `PlainTextResponse("ok")`. The endpoint does zero work to isolate the middleware overhead: any difference between configs is purely the cost of the middleware plumbing itself. Both middlewares are just calling the next layer. Same work, different base class.
Apache Bench (`ab`) fires requests at the server with 1,000 concurrent connections and a single uvicorn worker. One worker means one event loop, so the benchmark directly measures how each middleware design handles concurrent load on a single thread.
<BenchmarkVisualization />
<details>
<summary>Try it yourself</summary>
Save the script below as `benchmark_middleware.py`, then run:
```bash
# Terminal 1 — start the "before" server (1 ASGI + 1 BaseHTTPMiddleware)
python benchmark_middleware.py --middleware mixed
# Terminal 2 — benchmark it
ab -n 50000 -c 1000 http://localhost:8000/health
# Stop the server, then start the "after" server (2x pure ASGI)
python benchmark_middleware.py --middleware asgi
# Terminal 2 — benchmark again
ab -n 50000 -c 1000 http://localhost:8000/health
```
```python
import argparse
import uvicorn
from fastapi import FastAPI
from fastapi.responses import PlainTextResponse
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from starlette.types import ASGIApp, Receive, Scope, Send
class NoOpBaseHTTPMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
return await call_next(request)
class NoOpPureASGIMiddleware:
def __init__(self, app: ASGIApp) -> None:
self.app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
await self.app(scope, receive, send)
def create_app(middleware_type: str | None = None, layers: int = 2) -> FastAPI:
app = FastAPI()
@app.get("/health")
async def health():
return PlainTextResponse("ok")
if middleware_type == "mixed":
app.add_middleware(NoOpBaseHTTPMiddleware)
app.add_middleware(NoOpPureASGIMiddleware)
elif middleware_type == "asgi":
for _ in range(layers):
app.add_middleware(NoOpPureASGIMiddleware)
return app
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--middleware", choices=["asgi", "mixed"], default=None)
parser.add_argument("--layers", type=int, default=2)
parser.add_argument("--port", type=int, default=8000)
args = parser.parse_args()
app = create_app(middleware_type=args.middleware, layers=args.layers)
uvicorn.run(app, host="0.0.0.0", port=args.port, workers=1, log_level="warning")
```
</details>
---
## Our Change
Here's what we replaced it with:
```python
class PrometheusAuthMiddleware:
def __init__(self, app: ASGIApp) -> None:
self.app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http" or "/metrics" not in scope.get("path", ""):
await self.app(scope, receive, send)
return
if litellm.require_auth_for_metrics_endpoint is True:
request = Request(scope, receive)
api_key = request.headers.get("Authorization") or ""
try:
await user_api_key_auth(request=request, api_key=api_key)
except Exception as e:
# send 401 directly via ASGI protocol
...
return
await self.app(scope, receive, send)
```
For the 99.9% of requests that aren't hitting `/metrics`, the middleware is now one dict lookup, one string check, and one function call. No objects allocated, no tasks spawned.
It's important to evaluate if the tools you're using are the right fit for the job as your software grows and handles more responsiblity. We're now putting in a static analysis check to prevent this from happening again with any newly introduced middlewares. If we find the use case is necessary then that's okay and we'll reevalute but for everything LiteLLM needs to do at the moment it's not.
This middleware change was one part of a broader optimization effort on the LiteLLM proxy. Across all optimizations combined, we've measured about a **30% reduction in proxy overhead** over the past two weeks.
---
<a id="footnote-1"></a>
<sup>1</sup> [Starlette Middleware — BaseHTTPMiddleware](https://starlette.dev/middleware/#basehttpmiddleware)
<a id="footnote-2"></a>
<sup>2</sup> [Apache HTTP server benchmarking tool (`ab`)](https://httpd.apache.org/docs/2.4/programs/ab.html)

136
docs/my-website/blog/litellm_observatory/index.md Normal file
View file

@ -0,0 +1,136 @@
---
slug: litellm-observatory
title: "Improve release stability with 24 hour load tests"
date: 2026-02-06T10:00:00
authors:
- name: Alexsander Hamir
title: "Performance Engineer, LiteLLM"
url: https://www.linkedin.com/in/alexsander-baptista/
image_url: https://github.com/AlexsanderHamir.png
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "How we built a long-running, release-validation system to catch regressions before they reach users."
tags: [testing, observability, reliability, releases]
hide_table_of_contents: false
---
![LiteLLM Observatory](https://raw.githubusercontent.com/AlexsanderHamir/assets/main/Screenshot%202026-01-31%20175355.png)
# Improve release stability with 24 hour load tests
As LiteLLM adoption has grown, so have expectations around reliability, performance, and operational safety. Meeting those expectations requires more than correctness-focused tests, it requires validating how the system behaves over time, under real-world conditions.
This post introduces **LiteLLM Observatory**, a long-running release-validation system we built to catch regressions before they reach users.
---
## Why We Built the Observatory
LiteLLM operates at the intersection of external providers, long-lived network connections, and high-throughput workloads. While our unit and integration tests do an excellent job validating correctness, they are not designed to surface issues that only appear after extended operation.
A subtle lifecycle edge case discovered in v1.81.3 reinforced the need for stronger release validation in this area.
---
## A Real-World Lifecycle Edge Case
In v1.81.3, we shipped a fix for an HTTP client memory leak. The change passed unit and integration tests and behaved correctly in short-lived runs.
The issue that surfaced was not caused by a single incorrect line of logic, but by how multiple components interacted over time:
- A cached `httpx` client was configured with a 1-hour TTL
- When the cache expired, the underlying HTTP connection was closed as expected
- A higher-level client continued to hold a reference to that connection
- Subsequent requests failed with:
```
Cannot send a request, as the client has been closed
```
**Before (with bug):**
| Provider | Requests | Success | Failures | Fail % |
|----------|----------|---------|----------|--------|
| OpenAI | 720,000 | 432,000 | 288,000 | 40% |
| Azure | 692,000 | 415,200 | 276,800 | 40% |
**After (fixed):**
| Provider | Requests | Success | Failures | Fail % |
|----------|------------|-----------|----------|---------|
| OpenAI | 1,200,000 | 1,199,988 | 12 | 0.001% |
| Azure | 1,150,000 | 1,149,982 | 18 | 0.002% |
Our focus moving forward is on being the first to detect issues, even when they aren’t covered by unit tests. LiteLLM Observatory is designed to surface latency regressions, OOMs, and failure modes that only appear under real traffic patterns in **our own production deployments** during release validation.
---
### How the Observatory Works
[LiteLLM Observatory](https://github.com/BerriAI/litellm-observatory) is a testing service that runs long-running tests against our LiteLLM deployments. We trigger tests by sending API requests, and results are automatically sent to Slack when tests complete.
#### How Tests Run
1. **Start a Test**: We send a request to the Observatory API with:
- Which LiteLLM deployment to test (URL and API key)
- Which test to run (e.g., `TestOAIAzureRelease`)
- Test settings (which models to test, how long to run, failure thresholds)
2. **Smart Queueing**:
- The system checks whether we are attempting to run the exact same test more than once
- If a duplicate test is already running or queued, we receive an error to avoid wasting resources
- Otherwise, the test is added to a queue and runs when capacity is available (up to 5 tests can run concurrently by default)
3. **Instant Response**: The API responds immediately—we do not wait for the test to finish. Tests may run for hours, but the request itself completes in milliseconds.
4. **Background Execution**:
- The test runs in the background, issuing requests against our LiteLLM deployment
- It tracks request success and failure rates over time
- When the test completes, results are automatically posted to our Slack channel
#### Example: The OpenAI / Azure Reliability Test
The `TestOAIAzureRelease` test is designed to catch a class of bugs that only surface after sustained runtime:
- **Duration**: Runs continuously for 3 hours
- **Behavior**: Cycles through specified models (such as `gpt-4` and `gpt-3.5-turbo`), issuing requests continuously
- **Why 3 Hours**: This helps catch issues where HTTP clients degrade or fail after extended use (for example, a bug observed in LiteLLM v1.81.3)
- **Pass / Fail Criteria**: The test passes if fewer than 1% of requests fail. If the failure rate exceeds 1%, the test fails and we are notified in Slack
- **Key Detail**: The same HTTP client is reused for the entire run, allowing us to detect lifecycle-related bugs that only appear under prolonged reuse
#### When We Use It
- **Before Deployments**: Run tests before promoting a new LiteLLM version to production
- **Routine Validation**: Schedule regular runs (daily or weekly) to catch regressions early
- **Issue Investigation**: Run tests on demand when we suspect a deployment issue
- **Long-Running Failure Detection**: Identify bugs that only appear under sustained load, beyond what short smoke tests can reveal
### Complementing Unit Tests
Unit tests remain a foundational part of our development process. They are fast and precise, but they don’t cover:
- Real provider behavior
- Long-lived network interactions
- Resource lifecycle edge cases
- Time-dependent regressions
LiteLLM Observatory complements unit tests by validating the system as it actually runs in production-like environments.
---
### Looking Ahead
Reliability is an ongoing investment.
LiteLLM Observatory is one of several systems we’re building to continuously raise the bar on release quality and operational safety. As LiteLLM evolves, so will our validation tooling, informed by real-world usage and lessons learned.
We’ll continue to share those improvements openly as we go.

95
docs/my-website/blog/model_cost_map_incident/index.md Normal file
View file

@ -0,0 +1,95 @@
---
slug: model-cost-map-incident
title: "Incident Report: Invalid model cost map on main"
date: 2026-02-10T10:00:00
authors:
- name: Ishaan Jaffer
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/ishaanjaffer/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
tags: [incident-report, stability]
hide_table_of_contents: false
---
**Date:** January 27, 2026
**Duration:** ~20 minutes
**Severity:** Low
**Status:** Resolved
## Summary
A malformed JSON entry in `model_prices_and_context_window.json` was merged to `main` ([`562f0a0`](https://github.com/BerriAI/litellm/commit/562f0a028251750e3d75386bee0e630d9796d0df)). This caused LiteLLM to silently fall back to a stale local copy of the model cost map. Users on older package versions lost cost tracking for newer models only (e.g. `azure/gpt-5.2`). No LLM calls were blocked.
- **LLM calls and proxy routing:** No impact.
- **Cost tracking:** Impacted for newer models not present in the local backup. Older models were unaffected. The incident lasted ~20 minutes until the commit was reverted.
{/* truncate */}
---
## Background
The model cost map is not in the request path. It is used after the LLM response comes back, inside a try/catch, to calculate spend. A missing entry never blocks a call.
```mermaid
flowchart TD
A["1. litellm.completion() receives request
litellm/main.py"] --> B["2. Route to provider
litellm/litellm_core_utils/get_llm_provider_logic.py"]
B --> C["3. LLM returns response
litellm/main.py"]
C --> D["4. Post-call: look up model in cost map
litellm/cost_calculator.py"]
D -->|"found"| E["5a. Attach cost to response"]
D -->|"not found (try/catch)"| F["5b. Log warning, set cost=0"]
E --> G["6. Return response to caller"]
F --> G
style D fill:#fff3cd,stroke:#ffc107
style F fill:#fff3cd,stroke:#ffc107
style E fill:#d4edda,stroke:#28a745
style G fill:#d4edda,stroke:#28a745
```
Both paths return a response to the caller. When the cost map lookup fails, the only difference is `cost=0` on that request.
---
## Root cause
LiteLLM fetches the model cost map from GitHub `main` at import time. If the fetch fails, it falls back to a local backup bundled with the package. Before this incident, the fallback was completely silent -- no warning was logged.
A contributor PR introduced an extra `{` bracket, producing invalid JSON. The remote fetch failed with `JSONDecodeError`, triggering the silent fallback. Users on older package versions had backup files missing newer models.
**Timeline:**
1. Malformed JSON merged to `main`
2. LiteLLM installations fall back to local backup on next import
3. Users report `"This model isn't mapped yet"` for newer models
4. Bad commit identified and reverted (~20 minutes)
---
## Remediation
| # | Action | Status | Code |
|---|---|---|---|
| 1 | CI validation on `model_prices_and_context_window.json` | ✅ Done | [`test-model-map.yaml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test-model-map.yaml) |
| 2 | Warning log on fallback to local backup | ✅ Done | [`get_model_cost_map.py#L57-L68`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L57-L68) |
| 3 | `GetModelCostMap` class with integrity validation helpers | ✅ Done | [`get_model_cost_map.py#L24-L149`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L24-L149) |
| 4 | Resilience test suite (bad hosted map, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py#L150-L291`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L150-L291) |
| 5 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py#L213-L228`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L213-L228) |
Enterprises that require zero external dependencies at import time can set `LITELLM_LOCAL_MODEL_COST_MAP=True` to skip the GitHub fetch entirely.
---
## Other dependencies on external resources
| Dependency | Impact if unavailable | Fallback |
|---|---|---|
| Model cost map (GitHub) | Cost tracking for newer models | Local backup (now with warning) |
| JWT public keys (IDP/SSO) | Auth fails | None |
| OIDC UserInfo (IDP/SSO) | Auth fails | None |
| HuggingFace model API | HF provider calls fail | None |
| Ollama tags (localhost) | Ollama model list stale | Static list |

9
docs/my-website/docs/benchmarks.md
View file

@ -5,6 +5,13 @@ import Image from '@theme/IdealImage';
Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpoint.
## Setting Up a Fake OpenAI Endpoint
For load testing and benchmarking, you can use a fake OpenAI proxy server. LiteLLM provides:
1. **Hosted endpoint**: Use our free hosted fake endpoint at `https://exampleopenaiendpoint-production.up.railway.app/`
2. **Self-hosted**: Set up your own fake OpenAI proxy server using [github.com/BerriAI/example_openai_endpoint](https://github.com/BerriAI/example_openai_endpoint)
Use this config for testing:
```yaml
@ -12,7 +19,7 @@ model_list:
- model_name: "fake-openai-endpoint"
litellm_params:
model: openai/any
api_base: https://your-fake-openai-endpoint.com/chat/completions
api_base: https://exampleopenaiendpoint-production.up.railway.app/ # or your self-hosted endpoint
api_key: "test"
```

150
docs/my-website/docs/extras/contributing_code.md
View file

@ -1,27 +1,36 @@
# Contributing Code
## **Checklist before submitting a PR**
## Checklist before submitting a PR
Here are the core requirements for any PR submitted to LiteLLM
Here are the core requirements for any PR submitted to LiteLLM:
- [ ] Sign the Contributor License Agreement (CLA) - [see details](#contributor-license-agreement-cla)
- [ ] Add testing, **Adding at least 1 test is a hard requirement** - [see details](#2-adding-testing-to-your-pr)
- [ ] Ensure your PR passes the following tests:
- [ ] [Unit Tests](#3-running-unit-tests)
- [ ] [Formatting / Linting Tests](#35-running-linting-tests)
- [ ] Keep scope as isolated as possible. As a general rule, your changes should address 1 specific problem at a time
- [ ] Sign the [Contributor License Agreement (CLA)](#contributor-license-agreement-cla)
- [ ] Keep scope as isolated as possible — your changes should address **one specific problem** at a time
## **Contributor License Agreement (CLA)**
### Proxy (Backend) PRs
- [ ] Add testing — **at least 1 test is a hard requirement** ([details](#2-adding-tests))
- [ ] Ensure your PR passes:
- [ ] [Unit Tests](#3-running-unit-tests) — `make test-unit`
- [ ] [Formatting / Linting Tests](#4-running-linting-tests) — `make lint`
### UI PRs
- [ ] Ensure the UI builds successfully — `npm run build`
- [ ] Ensure all UI unit tests pass — `npm run test`
- [ ] If you are adding a **new component** or **new logic**, add corresponding tests
## Contributor License Agreement (CLA)
Before contributing code to LiteLLM, you must sign our [Contributor License Agreement (CLA)](https://cla-assistant.io/BerriAI/litellm). This is a legal requirement for all contributions to be merged into the main repository. The CLA helps protect both you and the project by clearly defining the terms under which your contributions are made.
**Important:** We strongly recommend reviewing and signing the CLA before starting work on your contribution to avoid any delays in the PR process. You can find the CLA [here](https://cla-assistant.io/BerriAI/litellm) and sign it through our CLA management system when you submit your first PR.
**Important:** We strongly recommend signing the CLA **before** starting work on your contribution to avoid delays in the review process. You can find and sign the CLA [here](https://cla-assistant.io/BerriAI/litellm).
## Quick start
---
## 1. Setup your local dev environment
## Proxy (Backend)
Here's how to modify the repo locally:
### 1. Setting up your local dev environment
Step 1: Clone the repo
@ -29,56 +38,53 @@ Step 1: Clone the repo
git clone https://github.com/BerriAI/litellm.git
```
Step 2: Install dev dependencies:
Step 2: Install dev dependencies
```shell
poetry install --with dev --extras proxy
```
That's it, your local dev environment is ready!
### 2. Adding tests
## 2. Adding Testing to your PR
- Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm).
- This directory mirrors the `litellm/` directory 1:1 and should **only** contain mocked tests.
- **Do not** add real LLM API calls to this directory.
- Add your test to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm)
#### File naming convention for `tests/test_litellm/`
- This directory 1:1 maps the the `litellm/` directory, and can only contain mocked tests.
- Do not add real llm api calls to this directory.
The test directory follows the same structure as `litellm/`:
### 2.1 File Naming Convention for `tests/test_litellm/`
The `tests/test_litellm/` directory follows the same directory structure as `litellm/`.
- `litellm/proxy/test_caching_routes.py` maps to `litellm/proxy/caching_routes.py`
- `test_{filename}.py` maps to `litellm/{filename}.py`
- `litellm/proxy/test_caching_routes.py` maps to `litellm/proxy/caching_routes.py`
## 3. Running Unit Tests
### 3. Running unit tests
run the following command on the root of the litellm directory
Run the following command from the root of the `litellm` directory:
```shell
make test-unit
```
## 3.5 Running Linting Tests
### 4. Running linting tests
run the following command on the root of the litellm directory
Run the following command from the root of the `litellm` directory:
```shell
make lint
```
LiteLLM uses mypy for linting. On ci/cd we also run `black` for formatting.
LiteLLM uses `mypy` for type checking. CI/CD also runs `black` for formatting.
## 4. Submit a PR with your changes!
### 5. Submit a PR
- push your fork to your GitHub repo
- submit a PR from there
- Push your changes to your fork on GitHub
- Open a Pull Request from your fork
## Advanced
---
### Building LiteLLM Docker Image
## UI
Some people might want to build the LiteLLM docker image themselves. Follow these instructions if you want to build / run the LiteLLM Docker Image yourself.
### 1. Setting up your local dev environment
Step 1: Clone the repo
@ -86,17 +92,72 @@ Step 1: Clone the repo
git clone https://github.com/BerriAI/litellm.git
```
Step 2: Build the Docker Image
Step 2: Navigate to the UI dashboard directory
Build using Dockerfile.non_root
```shell
cd ui/litellm-dashboard
```
Step 3: Install dependencies
```shell
npm install
```
Step 4: Start the development server
```shell
npm run dev
```
### 2. Adding tests
If you are adding a **new component** or **new logic**, you must add corresponding tests.
### 3. Running UI unit tests
```shell
npm run test
```
### 4. Building the UI
Ensure the UI builds successfully before submitting your PR:
```shell
npm run build
```
### 5. Submit a PR
- Push your changes to your fork on GitHub
- Open a Pull Request from your fork
---
## Advanced
### Building the LiteLLM Docker Image
Follow these instructions if you want to build and run the LiteLLM Docker image yourself.
Step 1: Clone the repo
```shell
git clone https://github.com/BerriAI/litellm.git
```
Step 2: Build the Docker image
Build using `Dockerfile.non_root`:
```shell
docker build -f docker/Dockerfile.non_root -t litellm_test_image .
```
Step 3: Run the Docker Image
Step 3: Run the Docker image
Make sure config.yaml is present in the root directory. This is your litellm proxy config file.
Make sure `config.yaml` is present in the root directory. This is your LiteLLM proxy config file.
```shell
docker run \
@ -107,18 +168,19 @@ docker run \
litellm_test_image \
--config /app/config.yaml --detailed_debug
```
### Running LiteLLM Proxy Locally
1. cd into the `proxy/` directory
### Running the LiteLLM Proxy Locally
```
1. Navigate to the `proxy/` directory:
```shell
cd litellm/litellm/proxy
```
2. Run the proxy
2. Run the proxy:
```shell
python3 proxy_cli.py --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
```

411
docs/my-website/docs/integrations/websearch_interception.md Normal file
View file

@ -0,0 +1,411 @@
# Web Search Integration
Enable transparent server-side web search execution for any LLM provider. LiteLLM automatically intercepts web search tool calls and executes them using your configured search provider (Perplexity, Tavily, etc.).
## Quick Start
### 1. Configure Web Search Interception
Add to your `config.yaml`:
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
callbacks:
- websearch_interception:
enabled_providers:
- openai
- minimax
- anthropic
search_tool_name: perplexity-search # Optional
search_tools:
- search_tool_name: perplexity-search
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITY_API_KEY
```
### 2. Use with Any Provider
```python
import litellm
response = await litellm.acompletion(
model="gpt-4o",
messages=[
{"role": "user", "content": "What's the weather in San Francisco today?"}
],
tools=[
{
"type": "function",
"function": {
"name": "litellm_web_search",
"description": "Search the web for information",
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string", "description": "Search query"}
},
"required": ["query"]
}
}
}
]
)
# Response includes search results automatically!
print(response.choices[0].message.content)
```
## How It Works
When a model makes a web search tool call, LiteLLM:
1. **Detects** the `litellm_web_search` tool call in the response
2. **Executes** the search using your configured search provider
3. **Makes a follow-up request** with the search results
4. **Returns** the final answer to the user
```mermaid
sequenceDiagram
participant User
participant LiteLLM
participant LLM as LLM Provider
participant Search as Search Provider
User->>LiteLLM: Request with web_search tool
LiteLLM->>LLM: Forward request
LLM-->>LiteLLM: Response with tool_call
Note over LiteLLM: Detect web search<br/>tool call
LiteLLM->>Search: Execute search
Search-->>LiteLLM: Search results
LiteLLM->>LLM: Follow-up with results
LLM-->>LiteLLM: Final answer
LiteLLM-->>User: Final answer with search results
```
**Result**: One API call from user → Complete answer with search results
## Supported Providers
Web search integration works with **all providers** that use:
- ✅ **Base HTTP Handler** (`BaseLLMHTTPHandler`)
- ✅ **OpenAI Completion Handler** (`OpenAIChatCompletion`)
### Providers Using Base HTTP Handler
| Provider | Status | Notes |
|----------|--------|-------|
| **OpenAI** | ✅ Supported | GPT-4, GPT-3.5, etc. |
| **Anthropic** | ✅ Supported | Claude models via HTTP handler |
| **MiniMax** | ✅ Supported | All MiniMax models |
| **Mistral** | ✅ Supported | Mistral AI models |
| **Cohere** | ✅ Supported | Command models |
| **Fireworks AI** | ✅ Supported | All Fireworks models |
| **Together AI** | ✅ Supported | All Together AI models |
| **Groq** | ✅ Supported | All Groq models |
| **Perplexity** | ✅ Supported | Perplexity models |
| **DeepSeek** | ✅ Supported | DeepSeek models |
| **xAI** | ✅ Supported | Grok models |
| **Hugging Face** | ✅ Supported | Inference API models |
| **OCI** | ✅ Supported | Oracle Cloud models |
| **Vertex AI** | ✅ Supported | Google Vertex AI models |
| **Bedrock** | ✅ Supported | AWS Bedrock models (converse_like route) |
| **Azure OpenAI** | ✅ Supported | Azure-hosted OpenAI models |
| **Sagemaker** | ✅ Supported | AWS Sagemaker models |
| **Databricks** | ✅ Supported | Databricks models |
| **DataRobot** | ✅ Supported | DataRobot models |
| **Hosted VLLM** | ✅ Supported | Self-hosted VLLM |
| **Heroku** | ✅ Supported | Heroku-hosted models |
| **RAGFlow** | ✅ Supported | RAGFlow models |
| **Compactif** | ✅ Supported | Compactif models |
| **Cometapi** | ✅ Supported | Comet API models |
| **A2A** | ✅ Supported | Agent-to-Agent models |
| **Bytez** | ✅ Supported | Bytez models |
### Providers Using OpenAI Handler
| Provider | Status | Notes |
|----------|--------|-------|
| **OpenAI** | ✅ Supported | Native OpenAI API |
| **Azure OpenAI** | ✅ Supported | Azure-hosted OpenAI |
| **OpenAI-Compatible** | ✅ Supported | Any OpenAI-compatible API |
## Configuration
### WebSearch Interception Parameters
| Parameter | Type | Required | Description | Example |
|-----------|------|----------|-------------|---------|
| `enabled_providers` | List[String] | Yes | List of providers to enable web search for | `[openai, minimax, anthropic]` |
| `search_tool_name` | String | No | Specific search tool from `search_tools` config. If not set, uses first available. | `perplexity-search` |
### Provider Values
Use these values in `enabled_providers`:
| Provider | Value | Provider | Value |
|----------|-------|----------|-------|
| OpenAI | `openai` | Anthropic | `anthropic` |
| MiniMax | `minimax` | Mistral | `mistral` |
| Cohere | `cohere` | Fireworks AI | `fireworks_ai` |
| Together AI | `together_ai` | Groq | `groq` |
| Perplexity | `perplexity` | DeepSeek | `deepseek` |
| xAI | `xai` | Hugging Face | `huggingface` |
| OCI | `oci` | Vertex AI | `vertex_ai` |
| Bedrock | `bedrock` | Azure | `azure` |
| Sagemaker | `sagemaker_chat` | Databricks | `databricks` |
| DataRobot | `datarobot` | VLLM | `hosted_vllm` |
| Heroku | `heroku` | RAGFlow | `ragflow` |
| Compactif | `compactif` | Cometapi | `cometapi` |
| A2A | `a2a` | Bytez | `bytez` |
## Search Providers
Configure which search provider to use. LiteLLM supports multiple search providers:
| Provider | `search_provider` Value | Environment Variable |
|----------|------------------------|----------------------|
| **Perplexity AI** | `perplexity` | `PERPLEXITYAI_API_KEY` |
| **Tavily** | `tavily` | `TAVILY_API_KEY` |
| **Exa AI** | `exa_ai` | `EXA_API_KEY` |
| **Parallel AI** | `parallel_ai` | `PARALLEL_AI_API_KEY` |
| **Google PSE** | `google_pse` | `GOOGLE_PSE_API_KEY`, `GOOGLE_PSE_ENGINE_ID` |
| **DataForSEO** | `dataforseo` | `DATAFORSEO_LOGIN`, `DATAFORSEO_PASSWORD` |
| **Firecrawl** | `firecrawl` | `FIRECRAWL_API_KEY` |
| **SearXNG** | `searxng` | `SEARXNG_API_BASE` (required) |
| **Linkup** | `linkup` | `LINKUP_API_KEY` |
See [Search Providers Documentation](../search/index.md) for detailed setup instructions.
## Complete Configuration Example
```yaml
model_list:
# OpenAI
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
# MiniMax
- model_name: minimax
litellm_params:
model: minimax/MiniMax-M2.1
api_key: os.environ/MINIMAX_API_KEY
# Anthropic
- model_name: claude
litellm_params:
model: anthropic/claude-sonnet-4-5
api_key: os.environ/ANTHROPIC_API_KEY
# Azure OpenAI
- model_name: azure-gpt4
litellm_params:
model: azure/gpt-4
api_base: https://my-azure.openai.azure.com
api_key: os.environ/AZURE_API_KEY
litellm_settings:
callbacks:
- websearch_interception:
enabled_providers:
- openai
- minimax
- anthropic
- azure
search_tool_name: perplexity-search
search_tools:
- search_tool_name: perplexity-search
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITY_API_KEY
- search_tool_name: tavily-search
litellm_params:
search_provider: tavily
api_key: os.environ/TAVILY_API_KEY
```
## Usage Examples
### Python SDK
```python
import litellm
# Configure callbacks
litellm.callbacks = ["websearch_interception"]
# Make completion with web search tool
response = await litellm.acompletion(
model="gpt-4o",
messages=[
{"role": "user", "content": "What are the latest AI news?"}
],
tools=[
{
"type": "function",
"function": {
"name": "litellm_web_search",
"description": "Search the web for current information",
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Search query"
}
},
"required": ["query"]
}
}
}
]
)
print(response.choices[0].message.content)
```
### Proxy Server
```bash
# Start proxy with config
litellm --config config.yaml
# Make request
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4o",
"messages": [
{"role": "user", "content": "What is the weather in San Francisco?"}
],
"tools": [
{
"type": "function",
"function": {
"name": "litellm_web_search",
"description": "Search the web",
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string"}
},
"required": ["query"]
}
}
}
]
}'
```
## How Search Tool Selection Works
1. **If `search_tool_name` is specified** → Uses that specific search tool
2. **If `search_tool_name` is not specified** → Uses first search tool in `search_tools` list
```yaml
search_tools:
- search_tool_name: perplexity-search # ← This will be used if no search_tool_name specified
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITY_API_KEY
- search_tool_name: tavily-search
litellm_params:
search_provider: tavily
api_key: os.environ/TAVILY_API_KEY
```
## Troubleshooting
### Web Search Not Working
1. **Check provider is enabled**:
```yaml
enabled_providers:
- openai # Make sure your provider is in this list
```
2. **Verify search tool is configured**:
```yaml
search_tools:
- search_tool_name: perplexity-search
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITY_API_KEY
```
3. **Check API keys are set**:
```bash
export PERPLEXITY_API_KEY=your-key
```
4. **Enable debug logging**:
```python
litellm.set_verbose = True
```
### Common Issues
**Issue**: Model returns tool_calls instead of final answer
- **Cause**: Provider not in `enabled_providers` list
- **Solution**: Add provider to `enabled_providers`
**Issue**: "No search tool configured" error
- **Cause**: No search tools in `search_tools` config
- **Solution**: Add at least one search tool configuration
**Issue**: "Invalid function arguments json string" error (MiniMax)
- **Cause**: Fixed in latest version - arguments weren't properly JSON serialized
- **Solution**: Update to latest LiteLLM version
## Related Documentation
- [Search Providers](../search/index.md) - Detailed search provider setup
- [Claude Code WebSearch](../tutorials/claude_code_websearch.md) - Using with Claude Code
- [Tool Calling](../completion/function_call.md) - General tool calling documentation
- [Callbacks](./custom_callback.md) - Custom callback documentation
## Technical Details
### Architecture
Web search integration is implemented as a custom callback (`WebSearchInterceptionLogger`) that:
1. **Pre-request Hook**: Converts native web search tools to LiteLLM standard format
2. **Post-response Hook**: Detects web search tool calls in responses
3. **Agentic Loop**: Executes searches and makes follow-up requests automatically
### Supported APIs
- ✅ **Chat Completions API** (OpenAI format)
- ✅ **Anthropic Messages API** (Anthropic format)
- ✅ **Streaming** (automatically converted)
- ✅ **Non-streaming**
### Response Format Detection
The handler automatically detects response format:
- **OpenAI format**: `tool_calls` in assistant message
- **Anthropic format**: `tool_use` blocks in content
### Performance
- **Latency**: Adds one additional LLM call (follow-up request with search results)
- **Caching**: Search results can be cached (depends on search provider)
- **Parallel Searches**: Multiple search queries executed in parallel
## Contributing
Found a bug or want to add support for a new provider? See our [Contributing Guide](https://github.com/BerriAI/litellm/blob/main/CONTRIBUTING.md).

5
docs/my-website/docs/load_test.md
View file

@ -4,8 +4,9 @@ import Image from '@theme/IdealImage';
## Locust Load Test LiteLLM Proxy
1. Add `fake-openai-endpoint` to your proxy config.yaml and start your litellm proxy
litellm provides a free hosted `fake-openai-endpoint` you can load test against
1. Add `fake-openai-endpoint` to your proxy config.yaml and start your litellm proxy.
LiteLLM provides a free hosted `fake-openai-endpoint` you can load test against. You can also self-host your own fake OpenAI proxy server using [github.com/BerriAI/example_openai_endpoint](https://github.com/BerriAI/example_openai_endpoint).
```yaml
model_list:

6
docs/my-website/docs/load_test_advanced.md
View file

@ -29,12 +29,16 @@ Tutorial on how to get to 1K+ RPS with LiteLLM Proxy on locust
**Note:** we're currently migrating to aiohttp which has 10x higher throughput. We recommend using the `openai/` provider for load testing.
:::tip Setting Up a Fake OpenAI Endpoint
You can use our hosted fake endpoint or self-host your own using [github.com/BerriAI/example_openai_endpoint](https://github.com/BerriAI/example_openai_endpoint).
:::
```yaml
model_list:
- model_name: "fake-openai-endpoint"
litellm_params:
model: openai/any
api_base: https://your-fake-openai-endpoint.com/chat/completions
api_base: https://exampleopenaiendpoint-production.up.railway.app/ # or your self-hosted endpoint
api_key: "test"
```

13
docs/my-website/docs/mcp.md
View file

@ -506,7 +506,14 @@ Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
- **Operation IDs**: Each operation should have a unique `operationId` (this becomes the tool name)
- **Parameters**: Request parameters should be properly documented with types and descriptions
## MCP Oauth
## MCP OAuth
LiteLLM supports OAuth 2.0 for MCP servers -- both interactive (PKCE) flows for user-facing clients and machine-to-machine (M2M) `client_credentials` for backend services.
See the **[MCP OAuth guide](./mcp_oauth.md)** for setup instructions, sequence diagrams, and a test server.
<details>
<summary>Detailed OAuth reference (click to expand)</summary>
LiteLLM v 1.77.6 added support for OAuth 2.0 Client Credentials for MCP servers.
@ -588,6 +595,8 @@ sequenceDiagram
See the official [MCP Authorization Flow](https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization#authorization-flow-steps) for additional reference.
</details>
## Forwarding Custom Headers to MCP Servers
@ -1486,7 +1495,7 @@ async with stdio_client(server_params) as (read, write):
**Q: How do I use OAuth2 client_credentials (machine-to-machine) with MCP servers behind LiteLLM?**
At the moment LiteLLM only forwards whatever `Authorization` header/value you configure for the MCP server; it does not issue OAuth2 tokens by itself. If your MCP requires the Client Credentials grant, obtain the access token directly from the authorization server and set that bearer token as the MCP server’s Authorization header value. LiteLLM does not yet fetch or refresh those machine-to-machine tokens on your behalf, but we plan to add first-class client_credentials support in a future release so the proxy can manage those tokens automatically.
LiteLLM supports automatic token management for the `client_credentials` grant. Configure `client_id`, `client_secret`, and `token_url` on your MCP server and LiteLLM will fetch, cache, and refresh tokens automatically. See the [MCP OAuth M2M guide](./mcp_oauth.md#machine-to-machine-m2m-auth) for setup instructions.
**Q: When I fetch an OAuth token from the LiteLLM UI, where is it stored?**

244
docs/my-website/docs/mcp_oauth.md Normal file
View file

@ -0,0 +1,244 @@
# MCP OAuth
LiteLLM supports two OAuth 2.0 flows for MCP servers:
| Flow | Use Case | How It Works |
|------|----------|--------------|
| **Interactive (PKCE)** | User-facing apps (Claude Code, Cursor) | Browser-based consent, per-user tokens |
| **Machine-to-Machine (M2M)** | Backend services, CI/CD, automated agents | `client_credentials` grant, proxy-managed tokens |
## Interactive OAuth (PKCE)
For user-facing MCP clients (Claude Code, Cursor), LiteLLM supports the full OAuth 2.0 authorization code flow with PKCE.
### Setup
```yaml title="config.yaml" showLineNumbers
mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
auth_type: oauth2
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
```
[**See Claude Code Tutorial**](./tutorials/claude_responses_api#connecting-mcp-servers)
### How It Works
```mermaid
sequenceDiagram
participant Browser as User-Agent (Browser)
participant Client as Client
participant LiteLLM as LiteLLM Proxy
participant MCP as MCP Server (Resource Server)
participant Auth as Authorization Server
Note over Client,LiteLLM: Step 1 – Resource discovery
Client->>LiteLLM: GET /.well-known/oauth-protected-resource/{mcp_server_name}/mcp
LiteLLM->>Client: Return resource metadata
Note over Client,LiteLLM: Step 2 – Authorization server discovery
Client->>LiteLLM: GET /.well-known/oauth-authorization-server/{mcp_server_name}
LiteLLM->>Client: Return authorization server metadata
Note over Client,Auth: Step 3 – Dynamic client registration
Client->>LiteLLM: POST /{mcp_server_name}/register
LiteLLM->>Auth: Forward registration request
Auth->>LiteLLM: Issue client credentials
LiteLLM->>Client: Return client credentials
Note over Client,Browser: Step 4 – User authorization (PKCE)
Client->>Browser: Open authorization URL + code_challenge + resource
Browser->>Auth: Authorization request
Note over Auth: User authorizes
Auth->>Browser: Redirect with authorization code
Browser->>LiteLLM: Callback to LiteLLM with code
LiteLLM->>Browser: Redirect back with authorization code
Browser->>Client: Callback with authorization code
Note over Client,Auth: Step 5 – Token exchange
Client->>LiteLLM: Token request + code_verifier + resource
LiteLLM->>Auth: Forward token request
Auth->>LiteLLM: Access (and refresh) token
LiteLLM->>Client: Return tokens
Note over Client,MCP: Step 6 – Authenticated MCP call
Client->>LiteLLM: MCP request with access token + LiteLLM API key
LiteLLM->>MCP: MCP request with Bearer token
MCP-->>LiteLLM: MCP response
LiteLLM-->>Client: Return MCP response
```
**Participants**
- **Client** -- The MCP-capable AI agent (e.g., Claude Code, Cursor, or another IDE/agent) that initiates OAuth discovery, authorization, and tool invocations on behalf of the user.
- **LiteLLM Proxy** -- Mediates all OAuth discovery, registration, token exchange, and MCP traffic while protecting stored credentials.
- **Authorization Server** -- Issues OAuth 2.0 tokens via dynamic client registration, PKCE authorization, and token endpoints.
- **MCP Server (Resource Server)** -- The protected MCP endpoint that receives LiteLLM's authenticated JSON-RPC requests.
- **User-Agent (Browser)** -- Temporarily involved so the end user can grant consent during the authorization step.
**Flow Steps**
1. **Resource Discovery**: The client fetches MCP resource metadata from LiteLLM's `.well-known/oauth-protected-resource` endpoint to understand scopes and capabilities.
2. **Authorization Server Discovery**: The client retrieves the OAuth server metadata (token endpoint, authorization endpoint, supported PKCE methods) through LiteLLM's `.well-known/oauth-authorization-server` endpoint.
3. **Dynamic Client Registration**: The client registers through LiteLLM, which forwards the request to the authorization server (RFC 7591). If the provider doesn't support dynamic registration, you can pre-store `client_id`/`client_secret` in LiteLLM (e.g., GitHub MCP) and the flow proceeds the same way.
4. **User Authorization**: The client launches a browser session (with code challenge and resource hints). The user approves access, the authorization server sends the code through LiteLLM back to the client.
5. **Token Exchange**: The client calls LiteLLM with the authorization code, code verifier, and resource. LiteLLM exchanges them with the authorization server and returns the issued access/refresh tokens.
6. **MCP Invocation**: With a valid token, the client sends the MCP JSON-RPC request (plus LiteLLM API key) to LiteLLM, which forwards it to the MCP server and relays the tool response.
See the official [MCP Authorization Flow](https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization#authorization-flow-steps) for additional reference.
## Machine-to-Machine (M2M) Auth
LiteLLM automatically fetches, caches, and refreshes OAuth2 tokens using the `client_credentials` grant. No manual token management required.
### Setup
You can configure M2M OAuth via the LiteLLM UI or `config.yaml`.
### UI Setup
Navigate to the **MCP Servers** page and click **+ Add New MCP Server**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/d1f1e89c-a789-4975-8846-b15d9821984a/ascreenshot_630800e00a2e4b598baabfc25efbabd3_text_export.jpeg)
Enter a name for your server and select **HTTP** as the transport type.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/2008c9d6-6093-4121-beab-1e52c71376aa/ascreenshot_516ffd6c7b524465a253a56048c3d228_text_export.jpeg)
Paste the MCP server URL.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/b0ee8b7d-6de8-492b-8962-287987feec29/ascreenshot_b3efca82078a4c6bb1453c58161909f9_text_export.jpeg)
Under **Authentication**, select **OAuth**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/e1597814-ff8e-40b9-9d7b-864dcdbe0910/ascreenshot_2097612712264d8f9e553f7ca9175fb0_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/f6ea5694-f28a-4bc3-9c9a-bb79f199bd65/ascreenshot_9be839f55b1b4f96bfe24030ba2c7f8d_text_export.jpeg)
Choose **Machine-to-Machine (M2M)** as the OAuth flow type. This is for server-to-server authentication using the `client_credentials` grant — no browser interaction required.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/9853310c-1d86-4628-bad1-7a391eca0e4d/ascreenshot_f302a286fa264fdd8d56db53b8f9395c_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/df64dc65-ef86-475d-adaf-12e227d5e873/ascreenshot_9e2f41d43a76435f918a00b52ffcc639_text_export.jpeg)
Fill in the **Client ID** and **Client Secret** provided by your OAuth provider.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/0de5a7bd-9898-4fc7-8843-b23dd5aac47f/ascreenshot_b9087aaa81a14b5b9c199929efc4a563_text_export.jpeg)
Enter the **Token URL** — this is the endpoint LiteLLM will call to fetch access tokens using `client_credentials`.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/0aea70f1-558c-4dca-91bc-1175fe1ddc89/ascreenshot_b3fcf8a1287e4e2d9a3d67c4a29f7bff_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/e842ef09-1fd7-47a6-909b-252d389f0abc/ascreenshot_2a87dad3624847e7ac370591d1d1aedd_text_export.jpeg)
Scroll down and review the server URL and all fields, then click **Create MCP Server**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/0857712b-4b53-40f8-8c1f-a4c72edaa644/ascreenshot_47be3fcd5de64ed391f70c1fb74a8bfc_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/9d961765-955f-4905-a3dc-1a446aa3b2cc/ascreenshot_43fd39d014224564bc6b35aced1fb6d3_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/3825d5fa-8fd1-4e71-b090-77ff0259c3f6/ascreenshot_2509a7ebd9bf421eb0e82f2553566745_text_export.jpeg)
Once created, open the server and navigate to the **MCP Tools** tab to verify that LiteLLM can connect and list available tools.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/8107e27b-5072-4675-8fd6-89b47692b1bd/ascreenshot_f774bc76138f430d808fb4482ebfcdca_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/ce94bb7b-c81b-4396-9939-178efb2cdfce/ascreenshot_28b838ab6ae34c76858454555c4c1d79_text_export.jpeg)
Select a tool (e.g. **echo**) to test it. Fill in the required parameters and click **Call Tool**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/c459c1d3-ec29-4211-9c28-37fbe7783bbc/ascreenshot_e9b138b3c2cc4440bb1a6f42ac7ae861_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/5438ac60-e0ac-4a79-bf6f-5594f160d3b5/ascreenshot_9133a17d26204c46bce497e74685c483_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/a8f6821b-3982-4b4d-9b25-70c8aff5ac31/ascreenshot_28d474d0e62545a482cff6128527883a_text_export.jpeg)
LiteLLM automatically fetches an OAuth token behind the scenes and calls the tool. The result confirms the M2M OAuth flow is working end-to-end.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/c3924549-a949-48d1-ac67-ab4c30475859/ascreenshot_8f6eca9d717f45478d50a881bd244bb3_text_export.jpeg)
### Config.yaml Setup
```yaml title="config.yaml" showLineNumbers
mcp_servers:
my_mcp_server:
url: "https://my-mcp-server.com/mcp"
auth_type: oauth2
client_id: os.environ/MCP_CLIENT_ID
client_secret: os.environ/MCP_CLIENT_SECRET
token_url: "https://auth.example.com/oauth/token"
scopes: ["mcp:read", "mcp:write"] # optional
```
### How It Works
1. On first MCP request, LiteLLM POSTs to `token_url` with `grant_type=client_credentials`
2. The access token is cached in-memory with TTL = `expires_in - 60s`
3. Subsequent requests reuse the cached token
4. When the token expires, LiteLLM fetches a new one automatically
```mermaid
sequenceDiagram
participant Client as Client
participant LiteLLM as LiteLLM Proxy
participant Auth as Authorization Server
participant MCP as MCP Server
Client->>LiteLLM: MCP request + LiteLLM API key
LiteLLM->>Auth: POST /oauth/token (client_credentials)
Auth->>LiteLLM: access_token (expires_in: 3600)
LiteLLM->>MCP: MCP request + Bearer token
MCP-->>LiteLLM: MCP response
LiteLLM-->>Client: MCP response
Note over LiteLLM: Token cached for subsequent requests
Client->>LiteLLM: Next MCP request
LiteLLM->>MCP: MCP request + cached Bearer token
MCP-->>LiteLLM: MCP response
LiteLLM-->>Client: MCP response
```
### Test with Mock Server
Use [BerriAI/mock-oauth2-mcp-server](https://github.com/BerriAI/mock-oauth2-mcp-server) to test locally:
```bash title="Terminal 1 - Start mock server" showLineNumbers
pip install fastapi uvicorn
python mock_oauth2_mcp_server.py # starts on :8765
```
```yaml title="config.yaml" showLineNumbers
mcp_servers:
test_oauth2:
url: "http://localhost:8765/mcp"
auth_type: oauth2
client_id: "test-client"
client_secret: "test-secret"
token_url: "http://localhost:8765/oauth/token"
```
```bash title="Terminal 2 - Start proxy and test" showLineNumbers
litellm --config config.yaml --port 4000
# List tools
curl http://localhost:4000/mcp-rest/tools/list \
-H "Authorization: Bearer sk-1234"
# Call a tool
curl http://localhost:4000/mcp-rest/tools/call \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{"name": "echo", "arguments": {"message": "hello"}}'
```
### Config Reference
| Field | Required | Description |
|-------|----------|-------------|
| `auth_type` | Yes | Must be `oauth2` |
| `client_id` | Yes | OAuth2 client ID. Supports `os.environ/VAR_NAME` |
| `client_secret` | Yes | OAuth2 client secret. Supports `os.environ/VAR_NAME` |
| `token_url` | Yes | Token endpoint URL |
| `scopes` | No | List of scopes to request |

251
docs/my-website/docs/mcp_public_internet.md Normal file
View file

@ -0,0 +1,251 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Exposing MCPs on the Public Internet
Control which MCP servers are visible to external callers (e.g., ChatGPT, Claude Desktop) vs. internal-only callers. This is useful when you want a subset of your MCP servers available publicly while keeping sensitive servers restricted to your private network.
## Overview
| Property | Details |
|-------|-------|
| Description | IP-based access control for MCP servers — external callers only see servers marked as public |
| Setting | `available_on_public_internet` on each MCP server |
| Network Config | `mcp_internal_ip_ranges` in `general_settings` |
| Supported Clients | ChatGPT, Claude Desktop, Cursor, OpenAI API, or any MCP client |
## How It Works
When a request arrives at LiteLLM's MCP endpoints, LiteLLM checks the caller's IP address to determine whether they are an **internal** or **external** caller:
1. **Extract the client IP** from the incoming request (supports `X-Forwarded-For` when configured behind a reverse proxy).
2. **Classify the IP** as internal or external by checking it against the configured private IP ranges (defaults to RFC 1918: `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `127.0.0.0/8`).
3. **Filter the server list**:
- **Internal callers** see all MCP servers (public and private).
- **External callers** only see servers with `available_on_public_internet: true`.
This filtering is applied at every MCP access point: the MCP registry, tool listing, tool calling, dynamic server routes, and OAuth discovery endpoints.
```mermaid
flowchart TD
A[Incoming MCP Request] --> B[Extract Client IP Address]
B --> C{Is IP in private ranges?}
C -->|Yes - Internal caller| D[Return ALL MCP servers]
C -->|No - External caller| E[Return ONLY servers with<br/>available_on_public_internet = true]
```
## Walkthrough
This walkthrough covers two flows:
1. **Adding a public MCP server** (DeepWiki) and connecting to it from ChatGPT
2. **Making an existing server private** (Exa) and verifying ChatGPT no longer sees it
### Flow 1: Add a Public MCP Server (DeepWiki)
DeepWiki is a free MCP server — a good candidate to expose publicly so AI gateway users can access it from ChatGPT.
#### Step 1: Create the MCP Server
Navigate to the MCP Servers page and click **"+ Add New MCP Server"**.
![Click Add New MCP Server](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/28cc27c2-d980-4255-b552-ebf542ef95be/ascreenshot_30a7e3c043834f1c87b69e6ffc5bba4f_text_export.jpeg)
The create dialog opens. Enter **"DeepWiki"** as the server name.
![Enter server name](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/8c733c38-310a-40ef-8a5b-7af91cc7f74f/ascreenshot_16df83fed5bd4683a22a042e07063cec_text_export.jpeg)
For the transport type dropdown, select **HTTP** since DeepWiki uses the Streamable HTTP transport.
![Select transport type](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/e473f603-d692-40c7-a218-866c2e1cb554/ascreenshot_e93997971f2f44beac6152786889addf_text_export.jpeg)
Now scroll down to the MCP Server URL field.
![Configure server](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/b08d3c1f-9279-45b6-8efb-f73008901da6/ascreenshot_ce0de66f230a41b0a454e76653429021_text_export.jpeg)
Enter the DeepWiki MCP URL: `https://mcp.deepwiki.com/mcp`.
![Enter MCP server URL](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/e59f8285-cfde-4c57-aa79-24244acc9160/ascreenshot_8d575c66dc614a4183212ba282d22b41_text_export.jpeg)
With the name, transport, and URL filled in, the basic server configuration is complete.
![Server URL configured](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/0f1af7ed-760d-4445-bdec-3da706d4eef4/ascreenshot_d7d6db69bc254ded871d14a71188a212_text_export.jpeg)
#### Step 2: Enable "Available on Public Internet"
Before creating, scroll down and expand the **Permission Management / Access Control** section. This is where you control who can see this server.
![Expand Permission Management](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/cc10dea2-6028-4a27-a33b-1b1b7212efb5/ascreenshot_0fdd152b862a4bf39973bc805ce64c57_text_export.jpeg)
Toggle **"Available on Public Internet"** on. This is the key setting — it tells LiteLLM that external callers (like ChatGPT connecting from the public internet) should be able to discover and use this server.
![Toggle Available on Public Internet](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/39c14543-c5ae-4189-8f85-9efc87135820/ascreenshot_9991f54910c24e21bba5c05ea4fa8e28_text_export.jpeg)
With the toggle enabled, click **"Create"** to save the server.
![Click Create](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/843be209-aade-44f4-98da-e55d1644854c/ascreenshot_8cfc90345a5f4d069b397e80d0a6e449_text_export.jpeg)
#### Step 3: Connect from ChatGPT
Now let's verify it works. Open ChatGPT and look for the MCP server icon to add a new connection. The endpoint to use is `<your-litellm-url>/mcp`.
![ChatGPT add MCP server](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/58b5f674-edf4-4156-a5fa-5fdc8ed5d7b9/ascreenshot_36735f7c37394e919793968794614126_text_export.jpeg)
In the dropdown, select **"Add an MCP server"** to configure a new connection.
![ChatGPT MCP server option](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/f89da8af-bc61-44a7-a765-f52733f4970d/ascreenshot_6410a917b782437eb558de3bfcd35ffd_text_export.jpeg)
ChatGPT asks for a server label. Give it a recognizable name like "LiteLLM".
![Enter server label](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/88505afe-07c1-4674-a89c-8035a5d05eb6/ascreenshot_143aefc38ddd4d3f9f5823ca2cc09bc2_text_export.jpeg)
Next, enter the Server URL. This should be your LiteLLM proxy's MCP endpoint — `<your-litellm-url>/mcp`.
![Enter LiteLLM MCP URL](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/9048be4a-7e40-43e7-9789-059fed2741a6/ascreenshot_e81232c17fd148f48f0ae552e9dc2a10_text_export.jpeg)
Paste your LiteLLM URL and confirm it looks correct.
![URL pasted](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/7707e796-e146-47c8-bce0-58e6f4076272/ascreenshot_0710dc58b8ed4d6887856b1388d59329_text_export.jpeg)
ChatGPT also needs authentication. Enter your LiteLLM API key in the authentication field so it can connect to the proxy.
![Enter API key](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/f6cfcb81-021d-4a41-94d7-d4eaf449d025/ascreenshot_d635865abfb64732a7278922f08dbcaa_text_export.jpeg)
Click **"Connect"** to establish the connection.
![Click Connect](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/1146b326-6f0c-4050-9729-af5c88e1bc81/ascreenshot_e19fb857e5394b9a9bf77b075b4fb620_text_export.jpeg)
ChatGPT connects and shows the available tools. Since both DeepWiki and Exa are currently marked as public, ChatGPT can see tools from both servers.
![ChatGPT shows available MCP tools](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/43ac56b7-9933-4762-903a-370fc52c79b5/ascreenshot_39073d6dc3bc4bb6a79d93365a26a4f8_text_export.jpeg)
---
### Flow 2: Make an Existing Server Private (Exa)
Now let's do the reverse — take an existing MCP server (Exa) that's currently public and restrict it to internal access only. After this change, ChatGPT should no longer see Exa's tools.
#### Step 1: Edit the Server
Go to the MCP Servers table and click on the Exa server to open its detail view.
![Exa server overview](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/65844f13-b1ec-4092-b3fd-b1cae3c0c833/ascreenshot_cc8ea435c5e14761a1394ca80fe817c0_text_export.jpeg)
Switch to the **"Settings"** tab to access the edit form.
![Click Settings](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/d5b65271-561e-4d2a-b832-96d32611f6e4/ascreenshot_a200942b17264c1eb7a3ffdb2c2141f5_text_export.jpeg)
The edit form loads with Exa's current configuration.
![Edit server](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/119184f6-f3cd-45b7-9cfa-0ea08de27020/ascreenshot_c39a793da03a4f0fb84b5ee829af9034_text_export.jpeg)
#### Step 2: Toggle Off "Available on Public Internet"
Scroll down and expand the **Permission Management / Access Control** section to find the public internet toggle.
![Expand permissions](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/bf7114cc-8741-4fa0-a39a-fe625482e88a/ascreenshot_8a987649c03e46558a2ec9a6f2f539a4_text_export.jpeg)
Toggle **"Available on Public Internet"** off. This will hide Exa from any caller outside your private network.
![Toggle off public internet](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/f36af5ad-028f-4bb1-aed1-43e38ff9b733/ascreenshot_9128364a049f489bb8483e18e5c88015_text_export.jpeg)
Click **"Save Changes"** to apply. The change takes effect immediately — no proxy restart needed.
![Save changes](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/126a71b3-02e1-4d61-a208-942b92e9ef25/ascreenshot_f349ef69e08044dd8e4903f4286b7b97_text_export.jpeg)
#### Step 3: Verify in ChatGPT
Go back to ChatGPT to confirm Exa is no longer visible. You'll need to reconnect for ChatGPT to re-fetch the tool list.
![ChatGPT verify](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/15518882-8b19-44d3-9bba-245aeb62b4b1/ascreenshot_f98f59c51e6543e1be4f3960ba375fc9_text_export.jpeg)
Open the MCP server settings and select to add or reconnect a server.
![Reconnect to server](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/784d3174-77c0-42e6-a059-4c906db8f72a/ascreenshot_d77db951b83e4b15a00373222712f6b5_text_export.jpeg)
Enter the same LiteLLM MCP URL as before.
![Reconnect URL](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/17ef5fb0-b240-4556-8d20-753d359b7fcf/ascreenshot_583466ce9e8f40d1ba0af8b1e7d04413_text_export.jpeg)
Set the server label.
![Reconnect name](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/d7907637-c957-4a3c-ab4f-1600ca9a70a0/ascreenshot_e429eea43f3f4b3ca4d3ac5a77fbde2d_text_export.jpeg)
Enter your API key for authentication.
![Reconnect key](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/9cfff77a-37aa-4ca6-8032-0b46c50f37e3/ascreenshot_250664183399496b8f5c9f86f576fc0b_text_export.jpeg)
Click **"Connect"** to re-establish the connection.
![Click Connect](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/686f6307-b4ae-448b-ac6c-2c9d7b4f6b57/ascreenshot_3f499d0812af42ab89fed103cc21c249_text_export.jpeg)
This time, only DeepWiki's tools appear — Exa is gone. LiteLLM detected that ChatGPT is calling from a public IP and filtered out Exa since it's no longer marked as public. Internal users on your private network would still see both servers.
![Only DeepWiki tools visible](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/667d79b6-75f9-4799-9315-0c176e7a5e34/ascreenshot_efa43050ac0b4445a09e542fa8f270ff_text_export.jpeg)
## Configuration Reference
### Per-Server Setting
<Tabs>
<TabItem value="ui" label="UI">
Toggle **"Available on Public Internet"** in the Permission Management section when creating or editing an MCP server.
</TabItem>
<TabItem value="config" label="config.yaml">
```yaml title="config.yaml" showLineNumbers
mcp_servers:
deepwiki:
url: https://mcp.deepwiki.com/mcp
available_on_public_internet: true # visible to external callers
exa:
url: https://exa.ai/mcp
auth_type: api_key
auth_value: os.environ/EXA_API_KEY
available_on_public_internet: false # internal only (default)
```
</TabItem>
<TabItem value="api" label="API">
```bash title="Create a public MCP server" showLineNumbers
curl -X POST <your-litellm-url>/v1/mcp/server \
-H "Authorization: Bearer sk-..." \
-H "Content-Type: application/json" \
-d '{
"server_name": "DeepWiki",
"url": "https://mcp.deepwiki.com/mcp",
"transport": "http",
"available_on_public_internet": true
}'
```
```bash title="Update an existing server" showLineNumbers
curl -X PUT <your-litellm-url>/v1/mcp/server \
-H "Authorization: Bearer sk-..." \
-H "Content-Type: application/json" \
-d '{
"server_id": "<server-id>",
"available_on_public_internet": false
}'
```
</TabItem>
</Tabs>
### Custom Private IP Ranges
By default, LiteLLM treats RFC 1918 private ranges as internal. You can customize this in the **Network Settings** tab under MCP Servers, or via config:
```yaml title="config.yaml" showLineNumbers
general_settings:
mcp_internal_ip_ranges:
- "10.0.0.0/8"
- "172.16.0.0/12"
- "192.168.0.0/16"
- "100.64.0.0/10" # Add your VPN/Tailscale range
```
When empty, the standard private ranges are used (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `127.0.0.0/8`).

144
docs/my-website/docs/pass_through/bedrock.md
View file

@ -556,3 +556,147 @@ for event in response.get("completion"):
print(completion)
```
## Using LangChain AWS SDK with LiteLLM
You can use the [LangChain AWS SDK](https://python.langchain.com/docs/integrations/chat/bedrock/) with LiteLLM Proxy to get cost tracking, load balancing, and other LiteLLM features.
### Quick Start
**1. Install LangChain AWS**:
```bash showLineNumbers
pip install langchain-aws
```
**2. Setup LiteLLM Proxy**:
Create a `config.yaml`:
```yaml showLineNumbers
model_list:
- model_name: claude-sonnet
litellm_params:
model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0
aws_region_name: us-east-1
custom_llm_provider: bedrock
```
Start the proxy:
```bash showLineNumbers
export AWS_ACCESS_KEY_ID="your-access-key"
export AWS_SECRET_ACCESS_KEY="your-secret-key"
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
**3. Use LangChain with LiteLLM**:
```python showLineNumbers
from langchain_aws import ChatBedrockConverse
from langchain_core.messages import HumanMessage
# Your LiteLLM API key
API_KEY = "Bearer sk-1234"
# Initialize ChatBedrockConverse pointing to LiteLLM proxy
llm = ChatBedrockConverse(
model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
endpoint_url="http://localhost:4000/bedrock",
region_name="us-east-1",
aws_access_key_id=API_KEY,
aws_secret_access_key="bedrock" # Any non-empty value works
)
# Invoke the model
messages = [HumanMessage(content="Hello, how are you?")]
response = llm.invoke(messages)
print(response.content)
```
### Advanced Example: PDF Document Processing with Citations
LangChain AWS SDK supports Bedrock's document processing features. Here's how to use it with LiteLLM:
```python showLineNumbers
import os
import json
from langchain_aws import ChatBedrockConverse
from langchain_core.messages import HumanMessage
# Your LiteLLM API key
API_KEY = "Bearer sk-1234"
def get_llm() -> ChatBedrockConverse:
"""Initialize LLM pointing to LiteLLM proxy"""
llm = ChatBedrockConverse(
model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
base_model_id="anthropic.claude-3-7-sonnet-20250219-v1:0",
endpoint_url="http://localhost:4000/bedrock",
region_name="us-east-1",
aws_access_key_id=API_KEY,
aws_secret_access_key="bedrock"
)
return llm
if __name__ == "__main__":
# Initialize the LLM
llm = get_llm()
# Read PDF file as bytes (Converse API requires raw bytes)
with open("your-document.pdf", "rb") as file:
file_bytes = file.read()
# Prepare messages with document attachment
messages = [
HumanMessage(content=[
{"text": "What is the policy number in this document?"},
{
"document": {
"format": "pdf",
"name": "PolicyDocument",
"source": {"bytes": file_bytes},
"citations": {"enabled": True}
}
}
])
]
# Invoke the LLM
response = llm.invoke(messages)
# Print response with citations
print(json.dumps(response.content, indent=4))
```
### Supported LangChain Features
All LangChain AWS features work with LiteLLM:
| Feature | Supported | Notes |
|---------|-----------|-------|
| Text Generation | ✅ | Full support |
| Streaming | ✅ | Use `stream()` method |
| Document Processing | ✅ | PDF, images, etc. |
| Citations | ✅ | Enable in document config |
| Tool Use | ✅ | Function calling support |
| Multi-modal | ✅ | Text + images + documents |
### Troubleshooting
**Issue**: `UnknownOperationException` error
**Solution**: Make sure you're using the correct endpoint URL format:
- ✅ Correct: `http://localhost:4000/bedrock`
- ❌ Wrong: `http://localhost:4000/bedrock/v2`
**Issue**: Authentication errors
**Solution**: Ensure your API key is in the correct format:
```python
aws_access_key_id="Bearer sk-1234" # Include "Bearer " prefix
```

22
docs/my-website/docs/providers/litellm_proxy.md
View file

@ -227,6 +227,28 @@ response = litellm.completion(
)
```
## OAuth2/JWT Authentication
If your LiteLLM Proxy requires OAuth2/JWT authentication (e.g., Azure AD, Keycloak, Okta), the SDK can automatically obtain and refresh tokens for you.
```python
import litellm
from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=AzureADCredential(),
scope="api://my-litellm-proxy/.default"
)
litellm.api_base = "https://my-proxy.example.com"
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello!"}]
)
```
[Learn more about SDK Proxy Authentication (OAuth2/JWT Auto-Refresh) →](../proxy_auth)
## Sending `tags` to LiteLLM Proxy
Tags allow you to categorize and track your API requests for monitoring, debugging, and analytics purposes. You can send tags as a list of strings to the LiteLLM Proxy using the `extra_body` parameter.

287
docs/my-website/docs/providers/perplexity.md
View file

@ -120,6 +120,293 @@ All models listed here https://docs.perplexity.ai/docs/model-cards are supported
## Agentic Research API (Responses API)
Requires v1.72.6+
### Using Presets
Presets provide optimized defaults for specific use cases. Start with a preset for quick setup:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
# Using the pro-search preset
response = responses(
model="perplexity/preset/pro-search",
input="What are the latest developments in AI?",
custom_llm_provider="perplexity",
)
print(response.output)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
1. Setup config.yaml
```yaml
model_list:
- model_name: perplexity-pro-search
litellm_params:
model: perplexity/preset/pro-search
api_key: os.environ/PERPLEXITY_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://0.0.0.0:4000/v1/responses \
-H "Content-Type: application/json" \
-H "Authorization: Bearer anything" \
-d '{
"model": "perplexity-pro-search",
"input": "What are the latest developments in AI?"
}'
```
</TabItem>
</Tabs>
### Using Third-Party Models
Access models from OpenAI, Anthropic, Google, xAI, and other providers through Perplexity's unified API:
<Tabs>
<TabItem value="openai" label="OpenAI">
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
input="Explain quantum computing in simple terms",
custom_llm_provider="perplexity",
max_output_tokens=500,
)
print(response.output)
```
</TabItem>
<TabItem value="anthropic" label="Anthropic">
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/anthropic/claude-3-5-sonnet-20241022",
input="Write a short story about a robot learning to paint",
custom_llm_provider="perplexity",
max_output_tokens=500,
)
print(response.output)
```
</TabItem>
<TabItem value="google" label="Google">
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/google/gemini-2.0-flash-exp",
input="Explain the concept of neural networks",
custom_llm_provider="perplexity",
max_output_tokens=500,
)
print(response.output)
```
</TabItem>
<TabItem value="xai" label="xAI">
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/xai/grok-2-1212",
input="What makes a good AI assistant?",
custom_llm_provider="perplexity",
max_output_tokens=500,
)
print(response.output)
```
</TabItem>
</Tabs>
### Web Search Tool
Enable web search capabilities to access real-time information:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
input="What's the weather in San Francisco today?",
custom_llm_provider="perplexity",
tools=[{"type": "web_search"}],
instructions="You have access to a web_search tool. Use it for questions about current events.",
)
print(response.output)
```
### Reasoning Effort (Responses API)
Control the reasoning effort level for reasoning-capable models:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-5.2",
input="Solve this complex problem step by step",
custom_llm_provider="perplexity",
reasoning={"effort": "high"}, # Options: low, medium, high
max_output_tokens=1000,
)
print(response.output)
```
### Multi-Turn Conversations
Use message arrays for multi-turn conversations with context:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/anthropic/claude-3-5-sonnet-20241022",
input=[
{"type": "message", "role": "system", "content": "You are a helpful assistant."},
{"type": "message", "role": "user", "content": "What are the latest AI developments?"},
],
custom_llm_provider="perplexity",
instructions="Provide detailed, well-researched answers.",
max_output_tokens=800,
)
print(response.output)
```
### Streaming Responses
Stream responses for real-time output:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
input="Tell me a story about space exploration",
custom_llm_provider="perplexity",
stream=True,
max_output_tokens=500,
)
for chunk in response:
if hasattr(chunk, 'type'):
if chunk.type == "response.output_text.delta":
print(chunk.delta, end="", flush=True)
```
### Supported Third-Party Models
| Provider | Model Name | Function Call |
|----------|------------|---------------|
| OpenAI | gpt-4o | `responses(model="perplexity/openai/gpt-4o", ...)` |
| OpenAI | gpt-4o-mini | `responses(model="perplexity/openai/gpt-4o-mini", ...)` |
| OpenAI | gpt-5.2 | `responses(model="perplexity/openai/gpt-5.2", ...)` |
| Anthropic | claude-3-5-sonnet-20241022 | `responses(model="perplexity/anthropic/claude-3-5-sonnet-20241022", ...)` |
| Anthropic | claude-3-5-haiku-20241022 | `responses(model="perplexity/anthropic/claude-3-5-haiku-20241022", ...)` |
| Google | gemini-2.0-flash-exp | `responses(model="perplexity/google/gemini-2.0-flash-exp", ...)` |
| Google | gemini-2.0-flash-thinking-exp | `responses(model="perplexity/google/gemini-2.0-flash-thinking-exp", ...)` |
| xAI | grok-2-1212 | `responses(model="perplexity/xai/grok-2-1212", ...)` |
| xAI | grok-2-vision-1212 | `responses(model="perplexity/xai/grok-2-vision-1212", ...)` |
### Available Presets
| Preset Name | Function Call |
|----------------|--------------------------------------------------------|
| fast-search | `responses(model="perplexity/preset/fast-search", ...)`|
| pro-search | `responses(model="perplexity/preset/pro-search", ...)` |
| deep-research | `responses(model="perplexity/preset/deep-research", ...)`|
### Complete Example
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
# Comprehensive example with multiple features
response = responses(
model="perplexity/openai/gpt-4o",
input="Research the latest developments in quantum computing and provide sources",
custom_llm_provider="perplexity",
tools=[
{"type": "web_search"},
{"type": "fetch_url"}
],
instructions="Use web_search to find relevant information and fetch_url to retrieve detailed content from sources. Provide citations for all claims.",
max_output_tokens=1000,
temperature=0.7,
)
print(f"Response ID: {response.id}")
print(f"Model: {response.model}")
print(f"Status: {response.status}")
print(f"Output: {response.output}")
print(f"Usage: {response.usage}")
```
:::info
For more information about passing provider-specific parameters, [go here](../completion/provider_specific_params.md)

35
docs/my-website/docs/proxy/admin_ui_sso.md
View file

@ -223,6 +223,7 @@ GENERIC_USER_FIRST_NAME_ATTRIBUTE = "first_name"
GENERIC_USER_LAST_NAME_ATTRIBUTE = "last_name"
GENERIC_USER_ROLE_ATTRIBUTE = "given_role"
GENERIC_USER_PROVIDER_ATTRIBUTE = "provider"
GENERIC_USER_EXTRA_ATTRIBUTES = "department,employee_id,manager" # comma-separated list of additional fields to extract from SSO response
GENERIC_CLIENT_STATE = "some-state" # if the provider needs a state parameter
GENERIC_INCLUDE_CLIENT_ID = "false" # some providers enforce that the client_id is not in the body
GENERIC_SCOPE = "openid profile email" # default scope openid is sometimes not enough to retrieve basic user info like first_name and last_name located in profile scope
@ -239,6 +240,40 @@ Use `GENERIC_USER_ROLE_ATTRIBUTE` to specify which attribute in the SSO token co
Nested attribute paths are supported (e.g., `claims.role` or `attributes.litellm_role`).
**Capturing Additional SSO Fields**
Use `GENERIC_USER_EXTRA_ATTRIBUTES` to extract additional fields from the SSO provider response beyond the standard user attributes (id, email, name, etc.). This is useful when you need to access custom organization-specific data (e.g., department, employee ID, groups) in your [custom SSO handler](./custom_sso.md).
```shell
# Comma-separated list of field names to extract
GENERIC_USER_EXTRA_ATTRIBUTES="department,employee_id,manager,groups"
```
**Accessing Extra Fields in Custom SSO Handler:**
```python
from litellm.proxy.management_endpoints.types import CustomOpenID
async def custom_sso_handler(userIDPInfo: CustomOpenID):
# Access the extra fields
extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {}
user_department = extra_fields.get("department")
employee_id = extra_fields.get("employee_id")
user_groups = extra_fields.get("groups", [])
# Use these fields for custom logic (e.g., team assignment, access control)
# ...
```
**Nested Field Paths:**
Dot notation is supported for nested fields:
```shell
GENERIC_USER_EXTRA_ATTRIBUTES="org_info.department,org_info.cost_center,metadata.employee_type"
```
- Set Redirect URI, if your provider requires it
- Set a redirect url = `<your proxy base url>/sso/callback`
```shell

11
docs/my-website/docs/proxy/config_settings.md
View file

@ -395,7 +395,7 @@ router_settings:
| ATHINA_API_KEY | API key for Athina service
| ATHINA_BASE_URL | Base URL for Athina service (defaults to `https://log.athina.ai`)
| AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key)
| AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **true**
| AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **false**
| AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024
| ANTHROPIC_API_KEY | API key for Anthropic service
| ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
@ -548,6 +548,10 @@ router_settings:
| DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL | Default embedding model for MCP semantic tool filtering. Default is "text-embedding-3-small"
| DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD | Default similarity threshold for MCP semantic tool filtering. Default is 0.3
| DEFAULT_MCP_SEMANTIC_FILTER_TOP_K | Default number of top results to return for MCP semantic tool filtering. Default is 10
| MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600
| MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200
| MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10
| MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS | Seconds to subtract from token expiry when computing cache TTL. Default is 60
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
@ -640,6 +644,7 @@ router_settings:
| GENERIC_TOKEN_ENDPOINT | Token endpoint for generic OAuth providers
| GENERIC_USER_DISPLAY_NAME_ATTRIBUTE | Attribute for user's display name in generic auth
| GENERIC_USER_EMAIL_ATTRIBUTE | Attribute for user's email in generic auth
| GENERIC_USER_EXTRA_ATTRIBUTES | Comma-separated list of additional fields to extract from generic SSO provider response (e.g., "department,employee_id,groups"). Accessible via `CustomOpenID.extra_fields` in custom SSO handlers. Supports dot notation for nested fields
| GENERIC_USER_FIRST_NAME_ATTRIBUTE | Attribute for user's first name in generic auth
| GENERIC_USER_ID_ATTRIBUTE | Attribute for user ID in generic auth
| GENERIC_USER_LAST_NAME_ATTRIBUTE | Attribute for user's last name in generic auth
@ -779,6 +784,7 @@ router_settings:
| LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LITELLM_ASYNCIO_QUEUE_MAXSIZE | Maximum size for asyncio queues (e.g. log queues, spend update queues, and cookbook examples such as realtime audio in `nova_sonic_realtime.py`). Bounds in-memory growth to prevent OOM. Default is 1000.
| LOGFIRE_TOKEN | Token for Logfire logging service
| LOGFIRE_BASE_URL | Base URL for Logfire logging service (useful for self hosted deployments)
| LOGGING_WORKER_CONCURRENCY | Maximum number of concurrent coroutine slots for the logging worker on the asyncio event loop. Default is 100. Setting too high will flood the event loop with logging tasks which will lower the overall latency of the requests.
@ -806,6 +812,7 @@ router_settings:
| MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0
| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 50. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
| MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH | Maximum header length for MCP semantic filter tools. Default is 150
| MAX_POLICY_ESTIMATE_IMPACT_ROWS | Maximum number of rows returned when estimating the impact of a policy. Default is 1000
| MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001
| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
| MISTRAL_API_BASE | Base URL for Mistral API. Default is https://api.mistral.ai
@ -822,6 +829,8 @@ router_settings:
| MICROSOFT_USER_ID_ATTRIBUTE | Field name for user ID in Microsoft SSO response. Default is `id`
| MICROSOFT_USER_LAST_NAME_ATTRIBUTE | Field name for user last name in Microsoft SSO response. Default is `surname`
| MICROSOFT_USERINFO_ENDPOINT | Custom userinfo endpoint URL for Microsoft SSO (overrides default Microsoft Graph userinfo endpoint)
| MODEL_COST_MAP_MAX_SHRINK_RATIO | Maximum allowed shrinkage ratio when validating a fetched model cost map against the local backup. Rejects the fetched map if it is smaller than this fraction of the backup. Default is 0.5
| MODEL_COST_MAP_MIN_MODEL_COUNT | Minimum number of models a fetched cost map must contain to be considered valid. Default is 50
| NO_DOCS | Flag to disable Swagger UI documentation
| NO_REDOC | Flag to disable Redoc documentation
| NO_PROXY | List of addresses to bypass proxy

12
docs/my-website/docs/proxy/custom_sso.md
View file

@ -142,6 +142,18 @@ async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:
f"No ID found for user. userIDPInfo.id is None {userIDPInfo}"
)
#################################################
# Access extra fields from SSO provider (requires GENERIC_USER_EXTRA_ATTRIBUTES env var)
# Example: Set GENERIC_USER_EXTRA_ATTRIBUTES="department,employee_id,groups"
extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {}
user_department = extra_fields.get("department")
employee_id = extra_fields.get("employee_id")
user_groups = extra_fields.get("groups", [])
print(f"User department: {user_department}") # noqa
print(f"Employee ID: {employee_id}") # noqa
print(f"User groups: {user_groups}") # noqa
#################################################
#################################################
# Run your custom code / logic here

46
docs/my-website/docs/proxy/forward_client_headers.md
View file

@ -6,6 +6,52 @@ Control which model groups can forward client headers to the underlying LLM prov
By default, LiteLLM does not forward client headers to LLM provider APIs for security reasons. However, you can selectively enable header forwarding for specific model groups using the `forward_client_headers_to_llm_api` setting.
## How it Works
LiteLLM does **not** forward all client headers to the LLM provider. Instead, it uses an **allowlist** approach — only headers matching specific rules are forwarded. This ensures sensitive headers (like your LiteLLM API key) are never accidentally sent to upstream providers.
```mermaid
sequenceDiagram
participant Client as Client (SDK / curl)
participant Proxy as LiteLLM Proxy
participant Filter as Header Filter (Allowlist)
participant LLM as LLM Provider (OpenAI, Anthropic, etc.)
Client->>Proxy: Request with all headers<br/>(Authorization, x-trace-id,<br/>x-custom-header, anthropic-beta, etc.)
Proxy->>Filter: Check forward_client_headers_to_llm_api<br/>setting for this model group
Note over Filter: Allowlist rules:<br/>1. Headers starting with "x-" ✅<br/>2. "anthropic-beta" ✅<br/>3. "x-stainless-*" ❌ (blocked)<br/>4. All other headers ❌ (blocked)
Filter-->>Proxy: Return only allowed headers
Proxy->>LLM: Request with filtered headers<br/>(x-trace-id, x-custom-header,<br/>anthropic-beta)
LLM-->>Proxy: Response
Proxy-->>Client: Response
```
### Header Allowlist Rules
The following rules determine which headers are forwarded (see [`_get_forwardable_headers`](https://github.com/litellm/litellm/blob/main/litellm/proxy/litellm_pre_call_utils.py) in `litellm/proxy/litellm_pre_call_utils.py`):
| Rule | Example | Forwarded? |
|---|---|---|
| Headers starting with `x-` | `x-trace-id`, `x-custom-header`, `x-request-source` | ✅ Yes |
| `anthropic-beta` header | `anthropic-beta: prompt-caching-2024-07-31` | ✅ Yes |
| Headers starting with `x-stainless-*` | `x-stainless-lang`, `x-stainless-arch` | ❌ No (causes OpenAI SDK issues) |
| Standard HTTP headers | `Authorization`, `Content-Type`, `Host` | ❌ No |
| Other provider headers | `Accept`, `User-Agent` | ❌ No |
### Additional Header Mechanisms
| Mechanism | Description | Reference |
|---|---|---|
| **`x-pass-` prefix** | Headers prefixed with `x-pass-` are always forwarded with the prefix stripped, regardless of settings. E.g., `x-pass-anthropic-beta: value` → `anthropic-beta: value`. Works for all pass-through endpoints. | [Source code](https://github.com/litellm/litellm/blob/main/litellm/passthrough/utils.py) |
| **`openai-organization`** | Forwarded only when `forward_openai_org_id: true` is set in `general_settings`. | [Forward OpenAI Org ID](#enable-globally) |
| **User information headers** | When `add_user_information_to_llm_headers: true`, LiteLLM adds `x-litellm-user-id`, `x-litellm-org-id`, etc. | [User Information Headers](#user-information-headers-optional) |
| **Vertex AI pass-through** | Uses a separate, stricter allowlist: only `anthropic-beta` and `content-type`. | [Source code](https://github.com/litellm/litellm/blob/main/litellm/constants.py) |
## Configuration
## Enable Globally

58
docs/my-website/docs/proxy/guardrails/custom_code_guardrail.md
View file

@ -61,15 +61,23 @@ curl -X POST http://localhost:4000/chat/completions \
### Function Signature
Your code must define an `apply_guardrail` function:
Your code must define an `apply_guardrail` function. It can be either sync or async:
```python
# Sync version
def apply_guardrail(inputs, request_data, input_type):
# inputs: see table below
# request_data: {"model": "...", "user_id": "...", "team_id": "...", "metadata": {...}}
# input_type: "request" or "response"
return allow() # or block() or modify()
# Async version (recommended when using HTTP primitives)
async def apply_guardrail(inputs, request_data, input_type):
response = await http_post("https://api.example.com/check", body={"text": inputs["texts"][0]})
if response["success"] and response["body"].get("flagged"):
return block("Content flagged")
return allow()
```
### `inputs` Parameter
@ -145,6 +153,29 @@ def apply_guardrail(inputs, request_data, input_type):
| `char_count(text)` | Count characters |
| `lower(text)` / `upper(text)` / `trim(text)` | String transforms |
### HTTP Requests (Async)
Make async HTTP requests to external APIs for additional validation or content moderation.
| Function | Description |
|----------|-------------|
| `await http_request(url, method, headers, body, timeout)` | General async HTTP request |
| `await http_get(url, headers, timeout)` | Async GET request |
| `await http_post(url, body, headers, timeout)` | Async POST request |
**Response format:**
```python
{
"status_code": 200, # HTTP status code
"body": {...}, # Response body (parsed JSON or string)
"headers": {...}, # Response headers
"success": True, # True if status code is 2xx
"error": None # Error message if request failed
}
```
**Note:** When using HTTP primitives, define your function as `async def apply_guardrail(...)` for non-blocking execution.
## Examples
### Block PII (SSN)
@ -213,6 +244,29 @@ def apply_guardrail(inputs, request_data, input_type):
return allow()
```
### Call External Moderation API (Async)
```python
async def apply_guardrail(inputs, request_data, input_type):
# Call an external moderation API
for text in inputs["texts"]:
response = await http_post(
"https://api.example.com/moderate",
body={"text": text, "user_id": request_data["user_id"]},
headers={"Authorization": "Bearer YOUR_API_KEY"},
timeout=10
)
if not response["success"]:
# API call failed - decide whether to allow or block
return allow()
if response["body"].get("flagged"):
return block(response["body"].get("reason", "Content flagged"))
return allow()
```
### Combine Multiple Checks
```python
@ -241,8 +295,8 @@ Custom code runs in a restricted environment:
- ❌ No `import` statements
- ❌ No file I/O
- ❌ No network access
- ❌ No `exec()` or `eval()`
- ✅ HTTP requests via built-in `http_request`, `http_get`, `http_post` primitives
- ✅ Only LiteLLM-provided primitives available
## Per-Request Usage

115
docs/my-website/docs/proxy/guardrails/guardrail_policies.md
View file

@ -1,3 +1,7 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# [Beta] Guardrail Policies
Use policies to group guardrails and control which ones run for specific teams, keys, or models.
@ -10,6 +14,9 @@ Use policies to group guardrails and control which ones run for specific teams,
## Quick Start
<Tabs>
<TabItem value="config" label="config.yaml">
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4
@ -43,6 +50,26 @@ policy_attachments:
scope: "*" # apply to all requests
```
</TabItem>
<TabItem value="ui" label="UI (LiteLLM Dashboard)">
**Step 1: Create a Policy**
Go to **Policies** tab and click **+ Create New Policy**. Fill in the policy name, description, and select guardrails to add.
![Enter policy name](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/4ba62cc8-d2c4-4af1-a526-686295466928/ascreenshot_401eab3e2081466e8f4d4ffa3bf7bff4_text_export.jpeg)
![Add a description for the policy](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/51685e47-1d94-4d9c-acb0-3c88dce9f938/ascreenshot_a5cd40066ff34afbb1e4089a3c93d889_text_export.jpeg)
![Select a parent policy to inherit from](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/1d96c3d3-187a-4f7c-97d2-6ac1f093d51e/ascreenshot_8a3af3b2210547dca3d4709df920d005_text_export.jpeg)
![Select guardrails to add to the policy](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/23781274-e600-4d5f-a8a6-4a2a977a166c/ascreenshot_a2a45d2c5d064c77ab7cb47b569ad9e9_text_export.jpeg)
![Click Create Policy to save](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/1d1ae8a8-daa5-451b-9fa2-c5b607ff6220/ascreenshot_218c2dd259714be4aa3c4e1894c96878_text_export.jpeg)
</TabItem>
</Tabs>
Response headers show what ran:
```
@ -58,6 +85,9 @@ x-litellm-applied-guardrails: pii_masking,prompt_injection
You have a global baseline, but want to add extra guardrails for a specific team.
<Tabs>
<TabItem value="config" label="config.yaml">
```yaml showLineNumbers title="config.yaml"
policies:
global-baseline:
@ -81,6 +111,30 @@ policy_attachments:
- finance # team alias from /team/new
```
</TabItem>
<TabItem value="ui" label="UI (LiteLLM Dashboard)">
**Option 1: Create a team-scoped attachment**
Go to **Policies** > **Attachments** tab and click **+ Create New Attachment**. Select the policy and the teams to scope it to.
![Select teams for the attachment](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/50e58f54-3bc3-477e-a106-e58cb65fde7e/ascreenshot_85d2e3d9d8d24842baced92fea170427_text_export.jpeg)
![Select the teams to attach the policy to](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/f24066bb-0a73-49fb-87b6-c65ad3ca5b2f/ascreenshot_242476fbdac447309f65de78b0ed9fdd_text_export.jpeg)
**Option 2: Attach from team settings**
Go to **Teams** > click on a team > **Settings** tab > under **Policies**, select the policies to attach.
![Open team settings and click Edit Settings](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/c31c3735-4f9d-4c6a-896b-186e97296940/ascreenshot_4749bb24ce5942cca462acc958fd3822_text_export.jpeg)
![Select policies to attach to this team](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/da8d5d7a-d975-4bfe-acd2-f41dcea29520/ascreenshot_835a33b6cec545cbb2987f017fbaff90_text_export.jpeg)
<Image img={require('../../../img/policy_team_attach.png')} />
</TabItem>
</Tabs>
Now the `finance` team gets `pii_masking` + `strict_compliance_check` + `audit_logger`, while everyone else just gets `pii_masking`.
## Remove guardrails for a specific team
@ -201,6 +255,60 @@ policy_attachments:
- "test-*" # key alias pattern
```
**Tag-based** (matches keys/teams by metadata tags, wildcards supported):
```yaml showLineNumbers title="config.yaml"
policy_attachments:
- policy: hipaa-compliance
tags:
- "healthcare"
- "health-*" # wildcard - matches health-team, health-dev, etc.
```
Tags are read from key and team `metadata.tags`. For example, a key created with `metadata: {"tags": ["healthcare"]}` would match the attachment above.
## Test Policy Matching
Debug which policies and guardrails apply for a given context. Use this to verify your policy configuration before deploying.
<Tabs>
<TabItem value="ui" label="UI (LiteLLM Dashboard)">
Go to **Policies** > **Test** tab. Enter a team alias, key alias, model, or tags and click **Test** to see which policies match and what guardrails would be applied.
<Image img={require('../../../img/policy_test_matching.png')} />
</TabItem>
<TabItem value="api" label="API">
```bash
curl -X POST "http://localhost:4000/policies/resolve" \
-H "Authorization: Bearer <your_api_key>" \
-H "Content-Type: application/json" \
-d '{
"tags": ["healthcare"],
"model": "gpt-4"
}'
```
Response:
```json
{
"effective_guardrails": ["pii_masking"],
"matched_policies": [
{
"policy_name": "hipaa-compliance",
"matched_via": "tag:healthcare",
"guardrails_added": ["pii_masking"]
}
]
}
```
</TabItem>
</Tabs>
## Config Reference
### `policies`
@ -233,14 +341,18 @@ policy_attachments:
scope: ...
teams: [...]
keys: [...]
models: [...]
tags: [...]
```
| Field | Type | Description |
|-------|------|-------------|
| `policy` | `string` | **Required.** Name of the policy to attach. |
| `scope` | `string` | Use `"*"` to apply globally. |
| `teams` | `list[string]` | Team aliases (from `/team/new`). |
| `teams` | `list[string]` | Team aliases (from `/team/new`). Supports `*` wildcard. |
| `keys` | `list[string]` | Key aliases (from `/key/generate`). Supports `*` wildcard. |
| `models` | `list[string]` | Model names. Supports `*` wildcard. |
| `tags` | `list[string]` | Tag patterns (from key/team `metadata.tags`). Supports `*` wildcard. |
### Response Headers
@ -248,6 +360,7 @@ policy_attachments:
|--------|-------------|
| `x-litellm-applied-policies` | Policies that matched this request |
| `x-litellm-applied-guardrails` | Guardrails that actually ran |
| `x-litellm-policy-sources` | Why each policy matched (e.g., `hipaa=tag:healthcare; baseline=scope:*`) |
## How it works

139
docs/my-website/docs/proxy/guardrails/policy_tags.md Normal file
View file

@ -0,0 +1,139 @@
# Tag-Based Policy Attachments
Apply guardrail policies automatically to any key or team that has a specific tag. Instead of attaching policies one-by-one, tag your keys and let the policy engine handle the rest.
**Example:** Your security team requires all healthcare-related keys to run PII masking and PHI detection. Tag those keys with `health`, create a single tag-based attachment, and every matching key gets the guardrails automatically.
## 1. Create a Policy with Guardrails
Navigate to **Policies** in the left sidebar. You'll see a list of existing policies along with their guardrails.
![Policies list page showing existing policies and the + Add New Policy button](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/d7aa1e1f-011e-40bf-a356-6dfe9d5d54f1/ascreenshot_8db95c231a7f4a79a36c2a98ba127542_text_export.jpeg)
Click **+ Add New Policy**. In the modal, enter a name for your policy (e.g., `high-risk-policy2`). You can also type to search existing policy names if you want to reference them.
![Create New Policy modal — enter the policy name and optional description](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/18f1ff69-9b83-4a98-9aad-9892a104d3ff/ascreenshot_1c6b85231cad4ec695750b53bbbda52c_text_export.jpeg)
Scroll down to **Guardrails to Add**. Click the dropdown to see all available guardrails configured on your proxy — select the ones this policy should enforce.
![Guardrails to Add dropdown showing available guardrails like OAI-moderation, phi-pre-guard, pii-pre-guard](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/55cedad7-9939-44a1-8644-a184cde82ab7/ascreenshot_eab4e55b82b8411893eccb6234d60b82_text_export.jpeg)
After selecting your guardrails, they appear as chips in the input field. The **Resolved Guardrails** section below shows the final set that will be applied (including any inherited from a parent policy).
![Selected guardrails shown as chips: testing-pl, phi-pre-guard, pii-pre-guard. Resolved Guardrails preview below.](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/c06d5b08-1c85-4715-b827-3e6864880428/ascreenshot_7a082e55f3ad425f9009346c68afae23_text_export.jpeg)
Click **Create Policy** to save.
![Click Create Policy to save the new policy](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/7e6eae64-4bba-4d72-b226-d1308ac576a8/ascreenshot_22d0ed686c594221bbbd2f40df214d75_text_export.jpeg)
## 2. Add a Tag Attachment for the Policy
After creating the policy, switch to the **Attachments** tab. This is where you define *where* the policy applies.
![Switch to the Attachments tab — shows the attachment table and scope documentation](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/871ae6d9-16d1-44e2-baf2-7bb8a9e72087/ascreenshot_76e124619d70462ea0e2fbb46ded1ac9_text_export.jpeg)
Click **+ Add New Attachment**. The Attachments page explains the available scopes: Global, Teams, Keys, Models, and **Tags**.
![Attachments page showing scope types including Tags — click + Add New Attachment](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/d45ab8bc-fc1e-425b-8a3f-44d18df810ec/ascreenshot_425824030f3144b7ab3c0ac570349b00_text_export.jpeg)
In the **Create Policy Attachment** modal, first select the policy you just created from the dropdown.
![Select the policy to attach from the dropdown (e.g., high-risk-policy2)](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/e0dcac40-e39c-4a6a-9d9c-4bbb9ec0ee91/ascreenshot_445b19894e0b466196a13e20c8e67f2d_text_export.jpeg)
Choose **Specific (teams, keys, models, or tags)** as the scope type. This expands the form to show fields for Teams, Keys, Models, and Tags.
![Select "Specific" scope type to reveal the Tags field](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/f685e02a-e22e-4c6c-9742-d5268746214b/ascreenshot_14d63d9d06dd4fc7854cfeb5e8d9ef85_text_export.jpeg)
Scroll down to the **Tags** field and type the tag to match — here we enter `health`. You can enter any string, or use a wildcard pattern like `health-*` to match all tags starting with `health-` (e.g., `health-team`, `health-dev`).
![Tags field with "health" entered. Supports wildcards like prod-* matching prod-us, prod-eu.](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/14581df7-732c-4ea5-b36d-58270b00e92c/ascreenshot_e734c81418f046549b61a84b9d352a29_text_export.jpeg)
## 3. Check the Impact of the Attachment
Before creating the attachment, click **Estimate Impact** to preview how many keys and teams would be affected. This is your blast-radius check — make sure the scope is what you expect before applying.
![Click Estimate Impact — the tag "health" is entered and ready to preview](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/6ccb81d7-3d11-48b0-b634-fc4d738aa530/ascreenshot_2eb89e6ff13a4b12b61004660a36c30c_text_export.jpeg)
The **Impact Preview** appears inline, showing exactly how many keys and teams would be affected. In this example: "This attachment would affect **1 key** and **0 teams**", with the key alias `hi` listed.
![Impact Preview showing "This attachment would affect 1 key and 0 teams." Keys: hi](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/8834d85a-2c15-48dd-8d6b-810cf11ee5c4/ascreenshot_d814b42ca9f34c23b0c2269bfa3e64fb_text_export.jpeg)
Once you're satisfied with the impact, click **Create Attachment** to save.
![Click Create Attachment to finalize](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/4a8918f2-eedb-4f49-a53b-4e46d0387d2a/ascreenshot_b08d490d836d4f46b4e5cbb14f61377a_text_export.jpeg)
The attachment now appears in the table with the policy name `high-risk-policy2` and tag `health` visible.
![Attachments table showing the new attachment with policy high-risk-policy2 and tag "health"](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/45867887-0aec-44a4-963b-b6cc6c302e3e/ascreenshot_981caeff98574ec89a8a53cd295e5043_text_export.jpeg)
## 4. Create a Key with the Tag
Navigate to **Virtual Keys** in the left sidebar. Click **+ Create New Key**.
![Virtual Keys page showing existing keys — click + Create New Key](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/4c1f9448-e590-4546-9357-6f68aa395b27/ascreenshot_4a7bc5be9e4347f3a9fe46f78d938d7c_text_export.jpeg)
Enter a key name and select a model. Then expand **Optional Settings** and scroll down to the **Tags** field.
![Create New Key modal — enter the key name](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/f84f7a2b-8057-4926-9f80-d68e437c77cf/ascreenshot_a277c8611b6e41059663b0759cd85cab_text_export.jpeg)
In the **Tags** field, type `health` and press Enter. This is the tag the policy engine will match against.
![Tags field in key creation — type "health" to add the tag](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/3ad3bf10-76d2-4f15-9a66-ed6c99bb25c4/ascreenshot_8a8773fb65fc49329cb1716da92b2723_text_export.jpeg)
The tag `health` now appears as a chip in the Tags field. Confirm your settings look correct.
![Tags field showing "health" selected with a checkmark](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/de3e58a9-6013-4d0c-882e-5517ea286684/ascreenshot_c7eef1736fce4aa894ac3b118b3800a2_text_export.jpeg)
Click **Create Key** at the bottom of the form.
![Click Create Key to generate the new virtual key with the health tag](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/51d419ea-ee80-4e24-8e93-b99a844881bc/ascreenshot_097d4564289943a88e30b5d2e3eab262_text_export.jpeg)
A dialog appears with your new virtual key. Click **Copy Virtual Key** — you'll need this to test in the next step.
![Save your Key dialog — click Copy Virtual Key to copy it to clipboard](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/e87a0cc1-4d12-4066-bfa2-973159808fd1/ascreenshot_7b616a7291d0497a9c61bdcdb59394d7_text_export.jpeg)
## 5. Test the Key and Validate the Policy is Applied
Navigate to **Playground** in the left sidebar to test the key interactively.
![Navigate to Playground from the sidebar](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/e6f8a3ee-e9e8-4107-93d1-bfca734c5ce9/ascreenshot_539bde38abe646e49148a912fff2d257_text_export.jpeg)
Under **Virtual Key Source**, select "Virtual Key" and paste the key you just copied into the input field.
![Paste the virtual key into the Playground configuration](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/a6612c4a-d499-4e54-8019-f54fde674ad9/ascreenshot_e85ebb9051554594bab0da57823fafad_text_export.jpeg)
Select a model from the **Select Model** dropdown.
![Select a model (e.g., bedrock-claude-opus-4.5) from the dropdown](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/325e330f-3eff-4c5e-b177-21916138a2f5/ascreenshot_693478f89c034e949e08f3ed0dd05120_text_export.jpeg)
Type a message and press Enter. If a guardrail blocks the request, you'll see it in the response. In this example, the `testing-pl` guardrail detected an email pattern and returned a 403 error — confirming the policy is working.
![Guardrail in action — the request was blocked with "Content blocked: email pattern detected"](https://colony-recorder.s3.amazonaws.com/files/2026-02-11/2cf16809-d2e5-4eae-a7dd-6a16dfcca7ce/ascreenshot_727d7d4ed20b4a52b2b41e39fd36eccb_text_export.jpeg)
**Using curl:**
You can also verify via the command line. The response headers confirm which policies and guardrails were applied:
```bash
curl -v http://localhost:4000/chat/completions \
-H "Authorization: Bearer <your-tagged-key>" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "say hi"}]
}'
```
Check the response headers:
```
x-litellm-applied-policies: high-risk-policy2
x-litellm-applied-guardrails: pii-pre-guard,phi-pre-guard,testing-pl
x-litellm-policy-sources: high-risk-policy2=tag:health
```
| Header | What it tells you |
|--------|-------------------|
| `x-litellm-applied-policies` | Which policies matched this request |
| `x-litellm-applied-guardrails` | Which guardrails actually ran |
| `x-litellm-policy-sources` | **Why** each policy matched — `tag:health` confirms it was the tag |

28
docs/my-website/docs/proxy/guardrails/zscaler_ai_guard.md
View file

@ -100,7 +100,7 @@ In cases where encounter other errors when apply Zscaler AI Guard, return exampl
}
}
```
## 6. Sending User Information to Zscaler AI Guard for Analysis (Optional)
## 6. Sending User Information to Zscaler AI Guard (Optional)
If you need to send end-user information to Zscaler AI Guard for analysis, you can set the configuration in the environment variables to True and include the relevant information in custom_headers on Zscaler AI Guard.
- To send user_api_key_alias:
@ -133,4 +133,30 @@ curl -i http://localhost:8165/v1/chat/completions \
"zguard_policy_id": <the custom policy id>
}
}'
```
## 8. Set Custom Zscaler AI Guard Policy on Litellm Team OR Key Metadata (Optional)
In addition to setting `zguard_policy_id` in a request or the configuration file, you can also set it in the metadata for LiteLLM Team or Key. The `zguard_policy_id` is determined using the following order of precedence: request, Key, Team, config file. This logic is illustrated below:
```
user_api_key_metadata = metadata.get("user_api_key_metadata", {}) or {}
team_metadata = metadata.get("team_metadata", {}) or {}
policy_id = (
metadata.get("zguard_policy_id")
if "zguard_policy_id" in metadata
else (
user_api_key_metadata.get("zguard_policy_id")
if "zguard_policy_id" in user_api_key_metadata
else (
team_metadata.get("zguard_policy_id")
if "zguard_policy_id" in team_metadata
else self.policy_id
)
)
)
```
You can leverage this feature to apply multiple policies configured on the Zscaler AI Guard (ZGuard) to traffic from different applications. (Note: It is recommended to map policies using either Team or Key metadata, but not a mix of both.)
Example set in Team/Key Metadata, you can set From UI:
```
{"zguard_policy_id": 100}
```

130
docs/my-website/docs/proxy/ui_team_soft_budget_alerts.md Normal file
View file

@ -0,0 +1,130 @@
import Image from '@theme/IdealImage';
# Team Soft Budget Alerts
Set a soft budget on a team and get email alerts when spending crosses the threshold — without blocking any requests.
## Overview
A **soft budget** is a spending threshold that triggers email notifications when exceeded, but **does not block requests**. This is different from a hard budget (`max_budget`), which rejects requests once the limit is reached.
<Image img={require('../../img/ui_team_soft_budget_alerts.png')} />
Team soft budget alerts let you:
- **Get notified early** — receive email alerts when a team's spend crosses the soft budget threshold
- **Keep requests flowing** — unlike hard budgets, soft budgets never block API calls
- **Target specific recipients** — send alerts to specific email addresses (e.g. team leads, finance), not just the team members
- **Work without global alerting** — team soft budget alerts are sent via email independently of Slack or other global alerting configuration
:::warning Email integration required
Team soft budget alerts are sent via email. You must have an active email integration (SendGrid, Resend, or SMTP) configured on your proxy for alerts to be delivered. See [Email Notifications](./email.md) for setup instructions.
:::
:::info Automatically active
Team soft budget alerts are **automatically active** once you configure a soft budget and at least one alerting email on a team. No additional proxy configuration or restart is needed — alerts are checked on every request.
:::
## How It Works
On every API request made with a key belonging to a team, the proxy checks:
1. Does the team have a `soft_budget` set?
2. Is the team's current `spend` >= the `soft_budget`?
3. Are there any emails configured in `soft_budget_alerting_emails`?
If all three conditions are met, an email alert is sent to the configured recipients. Alerts are **deduplicated** so the same alert is only sent once within a 24-hour window.
## How to Set Up Team Soft Budget Alerts
### 1. Navigate to the Admin UI
Go to the Admin UI (e.g. `http://localhost:4000/ui` or your `PROXY_BASE_URL/ui`).
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/f06d75ad-25ef-4ee8-90c3-9604f8e46a1c/ascreenshot_1a6defaed1494d6da0001459511ecfd5_text_export.jpeg)
### 2. Go to Teams
Click **Teams** in the sidebar.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/f06d75ad-25ef-4ee8-90c3-9604f8e46a1c/ascreenshot_2d258fa280f6463b966bf7a05bb102d5_text_export.jpeg)
### 3. Select a team
Click on the team you want to configure soft budget alerts for.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/490f09fb-6bf5-45a8-a384-676889f34c88/ascreenshot_15cceb22abe64df0bf7d7c742ecb5b2f_text_export.jpeg)
### 4. Open team Settings
Click the **Settings** tab to view the team's configuration.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/28dd1bc5-7d07-462f-b277-33f885bdc07e/ascreenshot_12f2b762b5d24686801d93ad5b067e06_text_export.jpeg)
### 5. Edit Settings
Click **Edit Settings** to modify the team's budget configuration.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/30a483ea-7e01-4fdc-ac5f-a5572388d138/ascreenshot_0915eadd9e754a798489853b82de3cb5_text_export.jpeg)
### 6. Set the Soft Budget
Click the **Soft Budget (USD)** field and enter your desired threshold. For example, enter `0.01` for testing or a higher value like `500` for production.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/8b306d80-4943-4ad0-a51a-94b5ebdd6680/ascreenshot_5bb6e65c6428473fac2607f6a7f4b98a_text_export.jpeg)
### 7. Add alerting emails
Click the **Soft Budget Alerting Emails** field and enter one or more comma-separated email addresses that should receive the alert.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/a97c6efa-cc93-45d7-979e-d2a533f423b9/ascreenshot_2d8223ce8e934aa1bfadfb2f78aee5fc_text_export.jpeg)
### 8. Save Changes
Click **Save Changes**. The soft budget alert is now active — no proxy restart required.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-07/865ba6f1-3fc6-4c19-8e08-433561d6c3f7/ascreenshot_b2f0503ada3a479a83dc8b7d01c1f8da_text_export.jpeg)
### 9. Verify: email alert received
Once the team's spend crosses the soft budget, an email alert is sent to the configured recipients. Below is an example of the alert email:
<Image img={require('../../img/ui_team_soft_budget_email_example.png')} />
## Settings Reference
| Setting | Description |
| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
| **Soft Budget (USD)** | The spending threshold that triggers an email alert. Requests are **not** blocked when this limit is exceeded. |
| **Soft Budget Alerting Emails** | Comma-separated email addresses that receive the alert when the soft budget is crossed. At least one email is required for alerts to be sent. |
:::tip Soft Budget vs. Max Budget
- **Soft Budget**: Advisory threshold — sends email alerts but does **not** block requests.
- **Max Budget**: Hard limit — blocks requests once the budget is exceeded.
You can set both on the same team to get early warnings (soft) and a hard stop (max).
:::
## API Configuration
You can also configure team soft budgets via the API when creating or updating a team:
```bash
curl -X POST 'http://localhost:4000/team/update' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"team_id": "your-team-id",
"soft_budget": 500.00,
"metadata": {
"soft_budget_alerting_emails": ["lead@example.com", "finance@example.com"]
}
}'
```
## Related Documentation
- [Email Notifications](./email.md) – Configure email integrations (Resend, SMTP) for LiteLLM Proxy
- [Alerting](./alerting.md) – Set up Slack and other alerting channels
- [Cost Tracking](./cost_tracking.md) – Track and manage spend across teams, keys, and users

333
docs/my-website/docs/proxy_auth.md Normal file
View file

@ -0,0 +1,333 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# SDK Proxy Authentication (OAuth2/JWT Auto-Refresh)
Automatically obtain and refresh OAuth2/JWT tokens when using the LiteLLM Python SDK with a LiteLLM Proxy that requires JWT authentication.
## Overview
When your LiteLLM Proxy is protected by an OAuth2/OIDC provider (Azure AD, Keycloak, Okta, Auth0, etc.), your SDK clients need valid JWT tokens for every request. Instead of manually managing token lifecycle, `litellm.proxy_auth` handles this automatically:
- Obtains tokens from your identity provider
- Caches tokens to avoid unnecessary requests
- Refreshes tokens before they expire (60-second buffer)
- Injects `Authorization: Bearer <token>` headers into every request
## Quick Start
### Azure AD
<Tabs>
<TabItem value="default" label="DefaultAzureCredential">
Uses the [DefaultAzureCredential](https://learn.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential) chain (environment variables, managed identity, Azure CLI, etc.):
```python
import litellm
from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
# One-time setup
litellm.proxy_auth = ProxyAuthHandler(
credential=AzureADCredential(), # uses DefaultAzureCredential
scope="api://my-litellm-proxy/.default"
)
litellm.api_base = "https://my-proxy.example.com"
# All requests now include Authorization headers automatically
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello!"}]
)
```
</TabItem>
<TabItem value="client-secret" label="ClientSecretCredential">
Use a specific Azure AD app registration:
```python
import litellm
from azure.identity import ClientSecretCredential
from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
azure_cred = ClientSecretCredential(
tenant_id="your-tenant-id",
client_id="your-client-id",
client_secret="your-client-secret"
)
litellm.proxy_auth = ProxyAuthHandler(
credential=AzureADCredential(credential=azure_cred),
scope="api://my-litellm-proxy/.default"
)
litellm.api_base = "https://my-proxy.example.com"
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello!"}]
)
```
</TabItem>
</Tabs>
**Required package:** `pip install azure-identity`
### Generic OAuth2 (Okta, Auth0, Keycloak, etc.)
Works with any OAuth2 provider that supports the `client_credentials` grant type:
```python
import litellm
from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=GenericOAuth2Credential(
client_id="your-client-id",
client_secret="your-client-secret",
token_url="https://your-idp.example.com/oauth2/token"
),
scope="litellm_proxy_api"
)
litellm.api_base = "https://my-proxy.example.com"
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello!"}]
)
```
### Custom Credential Provider
Implement the `TokenCredential` protocol to use any authentication mechanism:
```python
import time
import litellm
from litellm.proxy_auth import AccessToken, ProxyAuthHandler
class MyCustomCredential:
"""Any class with a get_token(scope) -> AccessToken method works."""
def get_token(self, scope: str) -> AccessToken:
# Your custom logic to obtain a token
token = my_auth_system.get_jwt(scope=scope)
return AccessToken(
token=token,
expires_on=int(time.time()) + 3600
)
litellm.proxy_auth = ProxyAuthHandler(
credential=MyCustomCredential(),
scope="my-scope"
)
```
## Supported Endpoints
Auth headers are automatically injected for:
| Endpoint | Function |
|----------|----------|
| Chat Completions | `litellm.completion()` / `litellm.acompletion()` |
| Embeddings | `litellm.embedding()` / `litellm.aembedding()` |
## How It Works
```
┌──────────┐ ┌──────────────────┐ ┌──────────────┐ ┌──────────────┐
│ Your │ │ ProxyAuthHandler │ │ Identity │ │ LiteLLM │
│ Code │────▶│ (token cache) │────▶│ Provider │ │ Proxy │
│ │ │ │◀────│ (Azure AD, │ │ │
│ │ │ │ │ Okta, etc) │ │ │
│ │ └────────┬─────────┘ └──────────────┘ │ │
│ │ │ Authorization: Bearer <token> │ │
│ │──────────────┼───────────────────────────────────▶│ │
│ │◀─────────────┼────────────────────────────────────│ │
└──────────┘ │ └──────────────┘
```
1. You set `litellm.proxy_auth` once at startup
2. On each SDK call (`completion()`, `embedding()`), the handler checks its cached token
3. If the token is missing or expires within 60 seconds, it requests a new one from your identity provider
4. The `Authorization: Bearer <token>` header is injected into the request
5. If token retrieval fails, a warning is logged and the request proceeds without auth headers
## API Reference
### ProxyAuthHandler
The main handler that manages the token lifecycle.
```python
from litellm.proxy_auth import ProxyAuthHandler
handler = ProxyAuthHandler(
credential=<TokenCredential>, # required - credential provider
scope="<oauth2-scope>" # required - OAuth2 scope to request
)
```
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `credential` | `TokenCredential` | Yes | A credential provider (AzureADCredential, GenericOAuth2Credential, or custom) |
| `scope` | `str` | Yes | The OAuth2 scope to request tokens for |
**Methods:**
| Method | Returns | Description |
|--------|---------|-------------|
| `get_token()` | `AccessToken` | Get a valid token, refreshing if needed |
| `get_auth_headers()` | `dict` | Get `{"Authorization": "Bearer <token>"}` headers |
### AzureADCredential
Wraps any `azure-identity` credential with lazy initialization.
```python
from litellm.proxy_auth import AzureADCredential
# Uses DefaultAzureCredential (recommended)
cred = AzureADCredential()
# Or wrap a specific azure-identity credential
from azure.identity import ManagedIdentityCredential
cred = AzureADCredential(credential=ManagedIdentityCredential())
```
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `credential` | Azure `TokenCredential` | No | An azure-identity credential. If `None`, uses `DefaultAzureCredential` |
### GenericOAuth2Credential
Standard OAuth2 client credentials flow for any provider.
```python
from litellm.proxy_auth import GenericOAuth2Credential
cred = GenericOAuth2Credential(
client_id="your-client-id",
client_secret="your-client-secret",
token_url="https://your-idp.com/oauth2/token"
)
```
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `client_id` | `str` | Yes | OAuth2 client ID |
| `client_secret` | `str` | Yes | OAuth2 client secret |
| `token_url` | `str` | Yes | Token endpoint URL |
### AccessToken
Dataclass representing an OAuth2 access token.
```python
from litellm.proxy_auth import AccessToken
token = AccessToken(
token="eyJhbG...", # JWT string
expires_on=1234567890 # Unix timestamp
)
```
### TokenCredential Protocol
Any class implementing this protocol can be used as a credential provider:
```python
from litellm.proxy_auth import AccessToken
class MyCredential:
def get_token(self, scope: str) -> AccessToken:
...
```
## Provider-Specific Examples
### Keycloak
```python
from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=GenericOAuth2Credential(
client_id="litellm-client",
client_secret="your-keycloak-client-secret",
token_url="https://keycloak.example.com/realms/your-realm/protocol/openid-connect/token"
),
scope="openid"
)
```
### Okta
```python
from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=GenericOAuth2Credential(
client_id="your-okta-client-id",
client_secret="your-okta-client-secret",
token_url="https://your-org.okta.com/oauth2/default/v1/token"
),
scope="litellm_api"
)
```
### Auth0
```python
from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=GenericOAuth2Credential(
client_id="your-auth0-client-id",
client_secret="your-auth0-client-secret",
token_url="https://your-tenant.auth0.com/oauth/token"
),
scope="https://my-proxy.example.com/api"
)
```
### Azure AD with Managed Identity
```python
from azure.identity import ManagedIdentityCredential
from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=AzureADCredential(
credential=ManagedIdentityCredential()
),
scope="api://my-litellm-proxy/.default"
)
```
## Combining with `use_litellm_proxy`
You can use `proxy_auth` together with [`use_litellm_proxy`](./providers/litellm_proxy#send-all-sdk-requests-to-litellm-proxy) to route all SDK requests through an authenticated proxy:
```python
import os
import litellm
from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
# Route all requests through the proxy
os.environ["LITELLM_PROXY_API_BASE"] = "https://my-proxy.example.com"
litellm.use_litellm_proxy = True
# Authenticate with OAuth2/JWT
litellm.proxy_auth = ProxyAuthHandler(
credential=AzureADCredential(),
scope="api://my-litellm-proxy/.default"
)
# This request goes through the proxy with automatic JWT auth
response = litellm.completion(
model="vertex_ai/gemini-2.0-flash-001",
messages=[{"role": "user", "content": "Hello!"}]
)
```

52
docs/my-website/docs/troubleshoot.md
View file

@ -45,6 +45,58 @@ Full error logs, stack traces, and any images from service metrics (CPU, memory,
---
## UI Issues
If you're experiencing issues with the LiteLLM Admin UI, please include the following information in addition to the general details above.
### 1. Steps to Reproduce
A clear, step-by-step description of how to trigger the issue (e.g., "Navigate to Settings → Team, click 'Create Team', fill in fields, click submit → error appears").
### 2. LiteLLM Version
The current version of LiteLLM you're running. Check via `litellm --version` or the UI's settings page.
### 3. Architecture & Deployment Setup
Distributed environments are a known source of UI issues. Please describe:
- **Number of LiteLLM instances/replicas** and how they are deployed (e.g., Kubernetes, Docker Compose, ECS)
- **Load balancer** type and configuration (e.g., ALB, Nginx, Cloudflare Tunnel) — include whether sticky sessions are enabled
- **How the UI is accessed** — directly via LiteLLM, through a reverse proxy, or behind an ingress controller
- **Any CDN or caching layers** between the user and the LiteLLM server
### 4. Network Tab Requests
Open your browser's Developer Tools (F12 → Network tab), reproduce the issue, and share:
- The **failing request(s)** — URL, method, status code, and response body
- **Screenshots or HAR export** of the relevant network activity
- Any **CORS or mixed-content errors** shown in the Console tab
### 5. Environment Variables
Non-sensitive environment variables related to the UI and proxy setup, such as:
- `LITELLM_MASTER_KEY`
- `PROXY_BASE_URL` / `LITELLM_PROXY_BASE_URL`
- `UI_BASE_PATH`
- Any SSO-related variables (e.g., `GOOGLE_CLIENT_ID`, `MICROSOFT_TENANT`)
Do **not** include passwords, secrets, or API keys.
### 6. Browser & Access Details
- **Browser** and version (e.g., Chrome 120, Firefox 121)
- **Access URL** used to reach the UI (redact sensitive parts)
- Whether the issue occurs for **all users or specific roles** (Admin, Internal User, etc.)
### 7. Screenshots or Screen Recordings
A screenshot or short screen recording of the issue is extremely helpful. Include any visible error messages, toasts, or unexpected behavior.
---
## Support Channels
[Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)

68
docs/my-website/docs/troubleshoot/max_callbacks.md Normal file
View file

@ -0,0 +1,68 @@
# MAX_CALLBACKS Limit
## Error Message
```
Cannot add callback - would exceed MAX_CALLBACKS limit of 30. Current callbacks: 30
```
## What This Means
LiteLLM limits the number of callbacks that can be registered to prevent performance degradation. Each callback runs on every LLM request, so having too many callbacks can cause exponential CPU usage and slow down your proxy.
The default limit is **30 callbacks**.
## When You Might Hit This Limit
- **Large enterprise deployments** with many teams, each having their own guardrails
- **Multiple logging integrations** combined with custom callbacks
- **Per-team callback configurations** that add up across your organization
## How to Override
Set the `LITELLM_MAX_CALLBACKS` environment variable to increase the limit:
```bash
# Docker
docker run -e LITELLM_MAX_CALLBACKS=100 ...
# Docker Compose
environment:
- LITELLM_MAX_CALLBACKS=100
# Kubernetes
env:
- name: LITELLM_MAX_CALLBACKS
value: "100"
# Direct
export LITELLM_MAX_CALLBACKS=100
litellm --config config.yaml
```
## Recommendations
1. **Start conservative** - Only increase as much as you need. If you have 60 teams with guardrails, try `LITELLM_MAX_CALLBACKS=75` to leave headroom.
2. **Monitor performance** - More callbacks means more processing per request. Watch your CPU usage and response latency after increasing the limit.
3. **Consolidate where possible** - If multiple teams use identical guardrails, consider using shared callback configurations rather than per-team duplicates.
## Example: Large Enterprise Setup
For an organization with 60+ teams, each with a guardrail callback:
```yaml
# config.yaml
litellm_settings:
callbacks: ["prometheus", "langfuse"] # 2 global callbacks
# Each team adds 1 guardrail callback = 60+ callbacks
# Total: 62+ callbacks needed
```
Set the environment variable:
```bash
export LITELLM_MAX_CALLBACKS=100
```

230
docs/my-website/docs/tutorials/claude_code_beta_headers.md Normal file
View file

@ -0,0 +1,230 @@
import Image from '@theme/IdealImage';
# Claude Code - Managing Anthropic Beta Headers
When using Claude Code with LiteLLM and non-Anthropic providers (Bedrock, Azure AI, Vertex AI), you need to ensure that only supported beta headers are sent to each provider. This guide explains how to add support for new beta headers or fix invalid beta header errors.
## What Are Beta Headers?
Anthropic uses beta headers to enable experimental features in Claude. When you use Claude Code, it may send beta headers like:
```
anthropic-beta: prompt-caching-scope-2026-01-05,advanced-tool-use-2025-11-20
```
However, not all providers support all Anthropic beta features. LiteLLM uses `anthropic_beta_headers_config.json` to manage which beta headers are supported by each provider.
## Common Error Message
```bash
Error: The model returned the following errors: invalid beta flag
```
## How LiteLLM Handles Beta Headers
LiteLLM uses a strict validation approach with a configuration file:
```
litellm/litellm/anthropic_beta_headers_config.json
```
This JSON file contains a **mapping** of beta headers for each provider:
- **Keys**: Input beta header names (from Anthropic)
- **Values**: Provider-specific header names (or `null` if unsupported)
- **Validation**: Only headers present in the mapping with non-null values are forwarded
This enforces stricter validation than just filtering unsupported headers - headers must be explicitly defined to be allowed.
## Adding Support for a New Beta Header
When Anthropic releases a new beta feature, you need to add it to the configuration file for each provider.
### Step 1: Locate the Config File
Find the file in your LiteLLM installation:
```bash
# If installed via pip
cd $(python -c "import litellm; import os; print(os.path.dirname(litellm.__file__))")
# The config file is at:
# litellm/anthropic_beta_headers_config.json
```
### Step 2: Add the New Beta Header
Open `anthropic_beta_headers_config.json` and add the new header to each provider's mapping:
```json title="anthropic_beta_headers_config.json"
{
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.",
"anthropic": {
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"new-feature-2026-03-01": "new-feature-2026-03-01",
...
},
"azure_ai": {
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"new-feature-2026-03-01": "new-feature-2026-03-01",
...
},
"bedrock_converse": {
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
"new-feature-2026-03-01": null,
...
},
"bedrock": {
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
"new-feature-2026-03-01": null,
...
},
"vertex_ai": {
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
"new-feature-2026-03-01": null,
...
}
}
```
**Key Points:**
- **Supported headers**: Set the value to the provider-specific header name (often the same as the key)
- **Unsupported headers**: Set the value to `null`
- **Header transformations**: Some providers use different header names (e.g., Bedrock maps `advanced-tool-use-2025-11-20` to `tool-search-tool-2025-10-19`)
- **Alphabetical order**: Keep headers sorted alphabetically for maintainability
### Step 3: Restart Your Application
After updating the config file, restart your LiteLLM proxy or application:
```bash
# If using LiteLLM proxy
litellm --config config.yaml
# If using Python SDK
# Just restart your Python application
```
The updated configuration will be loaded automatically.
## Fixing Invalid Beta Header Errors
If you encounter an "invalid beta flag" error, it means a beta header is being sent that the provider doesn't support.
### Step 1: Identify the Problematic Header
Check your logs to see which header is causing the issue:
```bash
Error: The model returned the following errors: invalid beta flag: new-feature-2026-03-01
```
### Step 2: Update the Config
Set the header value to `null` for that provider:
```json title="anthropic_beta_headers_config.json"
{
"bedrock_converse": {
"new-feature-2026-03-01": null
}
}
```
### Step 3: Restart and Test
Restart your application and verify the header is now filtered out.
## Contributing a Fix to LiteLLM
Help the community by contributing your fix!
### What to Include in Your PR
1. **Update the config file**: Add the new beta header to `litellm/anthropic_beta_headers_config.json`
2. **Test your changes**: Verify the header is correctly filtered/mapped for each provider
3. **Documentation**: Include provider documentation links showing which headers are supported
### Example PR Description
```markdown
## Add support for new-feature-2026-03-01 beta header
### Changes
- Added `new-feature-2026-03-01` to anthropic_beta_headers_config.json
- Set to `null` for bedrock_converse (unsupported)
- Set to header name for anthropic, azure_ai (supported)
### Testing
Tested with:
- ✅ Anthropic: Header passed through correctly
- ✅ Azure AI: Header passed through correctly
- ✅ Bedrock Converse: Header filtered out (returns error without fix)
### References
- Anthropic docs: [link]
- AWS Bedrock docs: [link]
```
## How Beta Header Filtering Works
When you make a request through LiteLLM:
```mermaid
sequenceDiagram
participant CC as Claude Code
participant LP as LiteLLM
participant Config as Beta Headers Config
participant Provider as Provider (Bedrock/Azure/etc)
CC->>LP: Request with beta headers
Note over CC,LP: anthropic-beta: header1,header2,header3
LP->>Config: Load header mapping for provider
Config-->>LP: Returns mapping (header→value or null)
Note over LP: Validate & Transform:<br/>1. Check if header exists in mapping<br/>2. Filter out null values<br/>3. Map to provider-specific names
LP->>Provider: Request with filtered & mapped headers
Note over LP,Provider: anthropic-beta: mapped-header2<br/>(header1, header3 filtered out)
Provider-->>LP: Success response
LP-->>CC: Response
```
### Filtering Rules
1. **Header must exist in mapping**: Unknown headers are filtered out
2. **Header must have non-null value**: Headers with `null` values are filtered out
3. **Header transformation**: Headers are mapped to provider-specific names (e.g., `advanced-tool-use-2025-11-20` → `tool-search-tool-2025-10-19` for Bedrock)
### Example
Request with headers:
```
anthropic-beta: advanced-tool-use-2025-11-20,computer-use-2025-01-24,unknown-header
```
For Bedrock Converse:
- ✅ `computer-use-2025-01-24` → `computer-use-2025-01-24` (supported, passed through)
- ❌ `advanced-tool-use-2025-11-20` → filtered out (null value in config)
- ❌ `unknown-header` → filtered out (not in config)
Result sent to Bedrock:
```
anthropic-beta: computer-use-2025-01-24
```
## Provider-Specific Notes
### Bedrock
- Beta headers appear in both HTTP headers AND request body (`additionalModelRequestFields.anthropic_beta`)
- Some headers are transformed (e.g., `advanced-tool-use` → `tool-search-tool`)
### Azure AI
- Uses same header names as Anthropic
- Some features not yet supported (check config for null values)
### Vertex AI
- Some headers are transformed to match Vertex AI's implementation
- Limited beta feature support compared to Anthropic

43
docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md Normal file
View file

@ -0,0 +1,43 @@
# Claude Code - Prompt Cache Routing
Claude's [Prompt Caching](https://platform.claude.com/docs/en/build-with-claude/prompt-caching) feature helps to optimize API usage through attempting to cache prompts and re-use cached prompts during subsequent API calls. This feature is used by Claude Code.
When LiteLLM [load balancing](../proxy/load_balancing.md) is enabled, to ensure this prompt caching feature still works with Claude Code, LiteLLM needs to be configured to use the `PromptCachingDeploymentCheck` pre-call check. This pre-call check will ensure that API calls that used prompt caching are remembered and that subsequent API calls that try to use that prompt caching are routed to the same model deployment where a cache write occurred.
## Set Up
1. Configure the router so that it uses the `PromptCachingDeploymentCheck` (via setting the `optional_pre_call_checks` property), and configure the models so that they can access multiple deployments of Claude; below, we show an example for multiple AWS accounts (referred to as `account-1` and `account-2`, using the `aws_profile_name` property):
```yaml
router_settings:
optional_pre_call_checks: ["prompt_caching"]
model_list:
- litellm_params:
model: us.anthropic.claude-sonnet-4-5-20250929-v1:0
aws_profile_name: account-1
aws_region_name: us-west-2
model_info:
litellm_provider: bedrock
model_name: us.anthropic.claude-sonnet-4-5-20250929-v1:0
- litellm_params:
model: us.anthropic.claude-sonnet-4-5-20250929-v1:0
aws_profile_name: account-2
aws_region_name: us-west-2
model_info:
litellm_provider: bedrock
model_name: us.anthropic.claude-sonnet-4-5-20250929-v1:0
```
2. Utilize Claude Code:
1. Launch Claude Code, which will do a warm-up API call that tries to cache its warm-up prompt and its system prompt.
2. Wait a few seconds, then quit Claude Code and re-open it.
3. You'll notice that the warm-up API call successfully gets a cache hit (if using Claude Code in an IDE like VS Code, ensure that you don't do anything between step 2.1 and 2.2 here, otherwise there may not be a cache hit):
1. Go to the [LiteLLM Request Logs page](../proxy/ui_logs.md) in the Admin UI
2. Click on the individual requests to see (a) the cache creation and cache read tokens; and (b) the Model ID. In particular, the API call from step 2.1 should show a cache write, and the API call from step 2.2 should show a cache read; in addition, the Model ID should be equal (meaning the API call is getting forwarded to the same AWS account).
## Related
- [Claude Code - Quickstart](./claude_responses_api.md)
- [Claude Code - Customer Tracking](./claude_code_customer_tracking.md)
- [Claude Code - Plugin Marketplace](./claude_code_plugin_marketplace.md)
- [Claude Code - WebSearch](./claude_code_websearch.md)
- [Proxy - Load Balancing](../proxy/load_balancing.md)

BIN
docs/my-website/img/policy_team_attach.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 225 KiB

BIN
docs/my-website/img/policy_test_matching.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 200 KiB

BIN
docs/my-website/img/release_notes/mcp_internet.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 724 KiB

BIN
docs/my-website/img/ui_team_soft_budget_alerts.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 328 KiB

BIN
docs/my-website/img/ui_team_soft_budget_email_example.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 50 KiB

2
docs/my-website/package.json
View file

@ -61,6 +61,8 @@
"mermaid": ">=11.10.0",
"gray-matter": "4.0.3",
"glob": ">=11.1.0",
"tar": ">=7.5.7",
"@isaacs/brace-expansion": ">=5.0.1",
"node-forge": ">=1.3.2",
"mdast-util-to-hast": ">=13.2.1",
"lodash-es": ">=4.17.23"

2
docs/my-website/release_notes/v1.81.3-stable/index.md
View file

@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.81.3.rc.2
docker.litellm.ai/berriai/litellm:v1.81.3-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.81.6.md
View file

@ -1,5 +1,5 @@
---
title: "v1.81.6 - Logs v2 with Tool Call Tracing"
title: "[Preview] v1.81.6 - Logs v2 with Tool Call Tracing"
slug: "v1-81-6"
date: 2026-01-31T00:00:00
authors:

372
docs/my-website/release_notes/v1.81.9.md Normal file
View file

@ -0,0 +1,372 @@
---
title: "[Preview] v1.81.9 - Control which MCP Servers are exposed on the Internet"
slug: "v1-81-9"
date: 2026-02-07T00:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
## Deploy this version
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.81.9.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.81.9
```
</TabItem>
</Tabs>
## Key Highlights
- **Claude Opus 4.6** - [Full support across Anthropic, AWS Bedrock, Azure AI, and Vertex AI with adaptive thinking and 1M context window](../../blog/claude_opus_4_6)
- **A2A Agent Gateway** - [Call A2A (Agent-to-Agent) registered agents through the standard `/chat/completions` API](../../docs/a2a_invoking_agents)
- **Expose MCP servers on the public internet** - [Launch MCP servers with public/private visibility and IP-based access control for internet-facing deployments](../../docs/mcp_public_internet)
- **UI Team Soft Budget Alerts** - [Set soft budgets on teams and receive email alerts when spending crosses the threshold — without blocking requests](../../docs/proxy/ui_team_soft_budget_alerts)
- **Performance Optimizations** - Multiple performance improvements including ~40% Prometheus CPU reduction, LRU caching, and optimized logging paths
- **LiteLLM Observatory** - [Automated 24-hour load tests](../../blog/litellm-observatory)
- **30% Faster Request Processing for Callback-Heavy Deployments** - [Performance improvement for callback heavy deployments][PR #20354](https://github.com/BerriAI/litellm/pull/20354)
---
## 30% Faster Request Processing for Callback-Heavy Deployments
If you use logging callbacks like Langfuse, Datadog, or Prometheus, every request was paying an unnecessary cost: three loops that re-sorted your callbacks on every single request, even though the callback list hadn't changed. The more callbacks you had configured, the more time was wasted. We moved this work to happen once at startup instead of on every request. For deployments with the default callback set, this is a ~30% speedup in request setup. For deployments with many callbacks configured, the improvement is even larger.
---
## LiteLLM Observatory
LiteLLM Observatory is a long-running release-validation system we built to catch regressions before they reach users. The system is built to be extensible—you can add new tests, configure models and failure thresholds, and queue runs against any deployment. Our goal is to achieve 100% coverage of LiteLLM functionality through these tests. We run 24-hour load tests against our production deployments before all releases, surfacing issues like resource lifecycle bugs, OOMs, and CPU regressions that only appear under sustained load.
---
## MCP Servers on the Public Internet
This release makes it safe to expose MCP servers on the public internet by adding public/private visibility and IP-based access control. You can now run internet-facing MCP services while restricting access to trusted networks and keeping internal tools private.
[Get started](../../docs/mcp_public_internet)
<Image
img={require('../img/release_notes/mcp_internet.png')}
style={{ maxWidth: '900px', width: '100%' }}
/>
## UI Team Soft Budget Alerts
Set a soft budget on any team to receive email alerts when spending crosses the threshold — without blocking any requests. Configure the threshold and alerting emails directly from the Admin UI, with no proxy restart needed.
[Get started](../../docs/proxy/ui_team_soft_budget_alerts)
<Image
img={require('../img/ui_team_soft_budget_alerts.png')}
style={{ maxWidth: '900px', width: '100%' }}
/>
Let's dive in.
---
## New Models / Updated Models
#### New Model Support (13 new models)
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) |
| -------- | ----- | -------------- | ------------------- | -------------------- |
| Anthropic | `claude-opus-4-6` | 1M | $5.00 | $25.00 |
| AWS Bedrock | `anthropic.claude-opus-4-6-v1` | 1M | $5.00 | $25.00 |
| Azure AI | `azure_ai/claude-opus-4-6` | 200K | $5.00 | $25.00 |
| Vertex AI | `vertex_ai/claude-opus-4-6` | 1M | $5.00 | $25.00 |
| Google Gemini | `gemini/deep-research-pro-preview-12-2025` | 65K | $2.00 | $12.00 |
| Vertex AI | `vertex_ai/deep-research-pro-preview-12-2025` | 65K | $2.00 | $12.00 |
| Moonshot | `moonshot/kimi-k2.5` | 262K | $0.60 | $3.00 |
| OpenRouter | `openrouter/qwen/qwen3-235b-a22b-2507` | 262K | $0.07 | $0.10 |
| OpenRouter | `openrouter/qwen/qwen3-235b-a22b-thinking-2507` | 262K | $0.11 | $0.60 |
| Together AI | `together_ai/zai-org/GLM-4.7` | 200K | $0.45 | $2.00 |
| Together AI | `together_ai/moonshotai/Kimi-K2.5` | 256K | $0.50 | $2.80 |
| ElevenLabs | `elevenlabs/eleven_v3` | - | $0.18/1K chars | - |
| ElevenLabs | `elevenlabs/eleven_multilingual_v2` | - | $0.18/1K chars | - |
#### Features
- **[Anthropic](../../docs/providers/anthropic)**
- Full Claude Opus 4.6 support with adaptive thinking across all regions (us, eu, apac, au) - [PR #20506](https://github.com/BerriAI/litellm/pull/20506), [PR #20508](https://github.com/BerriAI/litellm/pull/20508), [PR #20514](https://github.com/BerriAI/litellm/pull/20514), [PR #20551](https://github.com/BerriAI/litellm/pull/20551)
- Map reasoning content to anthropic thinking block (streaming + non-streaming) - [PR #20254](https://github.com/BerriAI/litellm/pull/20254)
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Add 1hr tiered caching costs for long-context models - [PR #20214](https://github.com/BerriAI/litellm/pull/20214)
- Support TTL (1h) field in prompt caching for Bedrock Claude 4.5 models - [PR #20338](https://github.com/BerriAI/litellm/pull/20338)
- Add Nova Sonic speech-to-speech model support - [PR #20244](https://github.com/BerriAI/litellm/pull/20244)
- Fix empty assistant message for Converse API - [PR #20390](https://github.com/BerriAI/litellm/pull/20390)
- Fix content blocked handling - [PR #20606](https://github.com/BerriAI/litellm/pull/20606)
- **[Google Gemini / Vertex AI](../../docs/providers/gemini)**
- Add Gemini Deep Research model support - [PR #20406](https://github.com/BerriAI/litellm/pull/20406)
- Fix Vertex AI Gemini streaming content_filter handling - [PR #20105](https://github.com/BerriAI/litellm/pull/20105)
- Allow using OpenAI-style tools for `web_search` with Vertex AI/Gemini models - [PR #20280](https://github.com/BerriAI/litellm/pull/20280)
- Fix `supports_native_streaming` for Gemini and Vertex AI models - [PR #20408](https://github.com/BerriAI/litellm/pull/20408)
- Add mapping for responses tools in file IDs - [PR #20402](https://github.com/BerriAI/litellm/pull/20402)
- **[Cohere](../../docs/providers/cohere)**
- Support `dimensions` param for Cohere embed v4 - [PR #20235](https://github.com/BerriAI/litellm/pull/20235)
- **[Cerebras](../../docs/providers/cerebras)**
- Add reasoning param support for GPT OSS Cerebras - [PR #20258](https://github.com/BerriAI/litellm/pull/20258)
- **[Moonshot](../../docs/providers/moonshot)**
- Add Kimi K2.5 model entries - [PR #20273](https://github.com/BerriAI/litellm/pull/20273)
- **[OpenRouter](../../docs/providers/openrouter)**
- Add Qwen3-235B models - [PR #20455](https://github.com/BerriAI/litellm/pull/20455)
- **[Together AI](../../docs/providers/togetherai)**
- Add GLM-4.7 and Kimi-K2.5 models - [PR #20319](https://github.com/BerriAI/litellm/pull/20319)
- **[ElevenLabs](../../docs/providers/elevenlabs)**
- Add `eleven_v3` and `eleven_multilingual_v2` TTS models - [PR #20522](https://github.com/BerriAI/litellm/pull/20522)
- **[Vercel AI Gateway](../../docs/providers/vercel_ai_gateway)**
- Add missing capability flags to models - [PR #20276](https://github.com/BerriAI/litellm/pull/20276)
- **[GitHub Copilot](../../docs/providers/github_copilot)**
- Fix system prompts being dropped and auto-add required Copilot headers - [PR #20113](https://github.com/BerriAI/litellm/pull/20113)
- **[GigaChat](../../docs/providers/gigachat)**
- Fix incorrect merging of consecutive user messages for GigaChat provider - [PR #20341](https://github.com/BerriAI/litellm/pull/20341)
- **[xAI](../../docs/providers/xai_realtime)**
- Add xAI `/realtime` API support - works with LiveKit SDK - [PR #20381](https://github.com/BerriAI/litellm/pull/20381)
- **[OpenAI](../../docs/providers/openai)**
- Add `gpt-5-search-api` model and docs clarifications - [PR #20512](https://github.com/BerriAI/litellm/pull/20512)
### Bug Fixes
- **[Anthropic](../../docs/providers/anthropic)**
- Fix extra inputs not permitted error for `provider_specific_fields` - [PR #20334](https://github.com/BerriAI/litellm/pull/20334)
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Fix: Managed Batches inconsistent state management for list and cancel batches - [PR #20331](https://github.com/BerriAI/litellm/pull/20331)
- **[OpenAI Embeddings](../../docs/providers/openai)**
- Fix `open_ai_embedding_models` to have `custom_llm_provider` None - [PR #20253](https://github.com/BerriAI/litellm/pull/20253)
---
## LLM API Endpoints
#### Features
- **[Messages API](../../docs/providers/anthropic)**
- Filter unsupported Claude Code beta headers for non-Anthropic providers - [PR #20578](https://github.com/BerriAI/litellm/pull/20578)
- Fix inconsistent response format in `anthropic.messages.acreate()` when using non-Anthropic providers - [PR #20442](https://github.com/BerriAI/litellm/pull/20442)
- Fix 404 on `/api/event_logging/batch` endpoint that caused Claude Code "route not found" errors - [PR #20504](https://github.com/BerriAI/litellm/pull/20504)
- **[A2A Agent Gateway](../../docs/a2a)**
- Allow calling A2A agents through LiteLLM `/chat/completions` API - [PR #20358](https://github.com/BerriAI/litellm/pull/20358)
- Use A2A registered agents with `/chat/completions` - [PR #20362](https://github.com/BerriAI/litellm/pull/20362)
- Fix A2A agents deployed with localhost/internal URLs in their agent cards - [PR #20604](https://github.com/BerriAI/litellm/pull/20604)
- **[Files API](../../docs/providers/gemini)**
- Add support for delete and GET via file_id for Gemini - [PR #20329](https://github.com/BerriAI/litellm/pull/20329)
- **General**
- Add User-Agent customization support - [PR #19881](https://github.com/BerriAI/litellm/pull/19881)
- Fix search tools not found when using per-request routers - [PR #19818](https://github.com/BerriAI/litellm/pull/19818)
- Forward extra headers in chat - [PR #20386](https://github.com/BerriAI/litellm/pull/20386)
---
## Management Endpoints / UI
#### Features
- **SSO Configuration**
- SSO Config Team Mappings - [PR #20111](https://github.com/BerriAI/litellm/pull/20111)
- UI - SSO: Add Team Mappings - [PR #20299](https://github.com/BerriAI/litellm/pull/20299)
- Extract user roles from JWT access token for Keycloak compatibility - [PR #20591](https://github.com/BerriAI/litellm/pull/20591)
- **Auth / SDK**
- Add `proxy_auth` for auto OAuth2/JWT token management in SDK - [PR #20238](https://github.com/BerriAI/litellm/pull/20238)
- **Virtual Keys**
- Key `reset_spend` endpoint - [PR #20305](https://github.com/BerriAI/litellm/pull/20305)
- UI - Keys: Allowed Routes to Key Info and Edit Pages - [PR #20369](https://github.com/BerriAI/litellm/pull/20369)
- Add Key info endpoint object permission data - [PR #20407](https://github.com/BerriAI/litellm/pull/20407)
- Keys and Teams Router Setting + Allow Override of Router Settings - [PR #20205](https://github.com/BerriAI/litellm/pull/20205)
- **Teams & Budgets**
- Add `soft_budget` to Team Table + Create/Update Endpoints - [PR #20530](https://github.com/BerriAI/litellm/pull/20530)
- Team Soft Budget Email Alerts - [PR #20553](https://github.com/BerriAI/litellm/pull/20553)
- UI - Team Settings: Soft Budget + Alerting Emails - [PR #20634](https://github.com/BerriAI/litellm/pull/20634)
- UI - User Budget Page: Unlimited Budget Checkbox - [PR #20380](https://github.com/BerriAI/litellm/pull/20380)
- `/user/update` allow for `max_budget` resets - [PR #20375](https://github.com/BerriAI/litellm/pull/20375)
- **UI Improvements**
- Default Team Settings: Migrate to use Reusable Model Select - [PR #20310](https://github.com/BerriAI/litellm/pull/20310)
- Navbar: Option to Hide Community Engagement Buttons - [PR #20308](https://github.com/BerriAI/litellm/pull/20308)
- Show team alias on Models health page - [PR #20359](https://github.com/BerriAI/litellm/pull/20359)
- Admin Settings: Add option for Authentication for public AI Hub - [PR #20444](https://github.com/BerriAI/litellm/pull/20444)
- Adjust daily spend date filtering for user timezone - [PR #20472](https://github.com/BerriAI/litellm/pull/20472)
- **SCIM**
- Add base `/scim/v2` endpoint for SCIM resource discovery - [PR #20301](https://github.com/BerriAI/litellm/pull/20301)
- **Proxy CLI**
- CLI arguments for RDS IAM auth - [PR #20437](https://github.com/BerriAI/litellm/pull/20437)
#### Bugs
- Fix: Remove unnecessary key blocking on UI login that prevented access - [PR #20210](https://github.com/BerriAI/litellm/pull/20210)
- UI - Team Settings: Disable Global Guardrail Persistence - [PR #20307](https://github.com/BerriAI/litellm/pull/20307)
- UI - Model Info Page: Fix Input and Output Labels - [PR #20462](https://github.com/BerriAI/litellm/pull/20462)
- UI - Model Page: Column Resizing on Smaller Screens - [PR #20599](https://github.com/BerriAI/litellm/pull/20599)
- Fix `/key/list` `user_id` Empty String Edge Case - [PR #20623](https://github.com/BerriAI/litellm/pull/20623)
- Add array type checks for model, agent, and MCP hub data to prevent UI crashes - [PR #20469](https://github.com/BerriAI/litellm/pull/20469)
- Fix unique constraint on daily tables + logging when updates fail - [PR #20394](https://github.com/BerriAI/litellm/pull/20394)
---
## Logging / Guardrail / Prompt Management Integrations
#### Bug Fixes (3 fixes)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Fix Langfuse OTEL trace export failing when spans contain null attributes - [PR #20382](https://github.com/BerriAI/litellm/pull/20382)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Fix incorrect failure metrics labels causing miscounted error rates - [PR #20152](https://github.com/BerriAI/litellm/pull/20152)
- **[Slack Alerts](../../docs/proxy/alerting)**
- Fix Slack alert delivery failing for certain budget threshold configurations - [PR #20257](https://github.com/BerriAI/litellm/pull/20257)
#### Guardrails (7 updates)
- **Custom Code Guardrails**
- Add HTTP support to custom code guardrails + Unified guardrails for MCP + Agent guardrail support - [PR #20619](https://github.com/BerriAI/litellm/pull/20619)
- Custom Code Guardrails UI Playground - [PR #20377](https://github.com/BerriAI/litellm/pull/20377)
- **Team-Based Guardrails**
- Implement team-based isolation guardrails management - [PR #20318](https://github.com/BerriAI/litellm/pull/20318)
- **[OpenAI Moderations](../../docs/apply_guardrail)**
- Ensure OpenAI Moderations Guard works with OpenAI Embeddings - [PR #20523](https://github.com/BerriAI/litellm/pull/20523)
- **[GraySwan / Cygnal](../../docs/apply_guardrail)**
- Fix fail-open for GraySwan and pass metadata to Cygnal API endpoint - [PR #19837](https://github.com/BerriAI/litellm/pull/19837)
- **General**
- Check for `model_response_choices` before guardrail input - [PR #19784](https://github.com/BerriAI/litellm/pull/19784)
- Preserve streaming content on guardrail-sampled chunks - [PR #20027](https://github.com/BerriAI/litellm/pull/20027)
---
## Spend Tracking, Budgets and Rate Limiting
- **Support 0 cost models** - Allow zero-cost model entries for internal/free-tier models - [PR #20249](https://github.com/BerriAI/litellm/pull/20249)
---
## MCP Gateway (9 updates)
- **MCP Semantic Filtering** - Filter MCP tools using semantic similarity to reduce tool sprawl for LLM calls - [PR #20296](https://github.com/BerriAI/litellm/pull/20296), [PR #20316](https://github.com/BerriAI/litellm/pull/20316)
- **UI - MCP Semantic Filtering** - Add support for MCP Semantic Filtering configuration on UI - [PR #20454](https://github.com/BerriAI/litellm/pull/20454)
- **MCP IP-Based Access Control** - Set MCP servers as private/public available on internet with IP-based restrictions - [PR #20607](https://github.com/BerriAI/litellm/pull/20607), [PR #20620](https://github.com/BerriAI/litellm/pull/20620)
- **Fix MCP "Session not found" error** on VSCode reconnect - [PR #20298](https://github.com/BerriAI/litellm/pull/20298)
- **Fix OAuth2 'Capabilities: none' bug** for upstream MCP servers - [PR #20602](https://github.com/BerriAI/litellm/pull/20602)
- **Include Config Defined Search Tools** in `/search_tools/list` - [PR #20371](https://github.com/BerriAI/litellm/pull/20371)
- **UI - Search Tools**: Show Config Defined Search Tools - [PR #20436](https://github.com/BerriAI/litellm/pull/20436)
- **Ensure MCP permissions are enforced** when using JWT Auth - [PR #20383](https://github.com/BerriAI/litellm/pull/20383)
- **Fix `gcs_bucket_name` not being passed** correctly for MCP server storage configuration - [PR #20491](https://github.com/BerriAI/litellm/pull/20491)
---
## Performance / Loadbalancing / Reliability improvements (14 improvements)
- **Prometheus ~40% CPU reduction** - Parallelize budget metrics, fix caching bug, reduce CPU usage - [PR #20544](https://github.com/BerriAI/litellm/pull/20544)
- **Prevent closed client errors** by reverting httpx client caching - [PR #20025](https://github.com/BerriAI/litellm/pull/20025)
- **Avoid unnecessary Router creation** when no models or search tools are configured - [PR #20661](https://github.com/BerriAI/litellm/pull/20661)
- **Optimize `wrapper_async`** with `CallTypes` caching and reduced lookups - [PR #20204](https://github.com/BerriAI/litellm/pull/20204)
- **Cache `_get_relevant_args_to_use_for_logging()`** at module level - [PR #20077](https://github.com/BerriAI/litellm/pull/20077)
- **LRU cache for `normalize_request_route`** - [PR #19812](https://github.com/BerriAI/litellm/pull/19812)
- **Optimize `get_standard_logging_metadata`** with set intersection - [PR #19685](https://github.com/BerriAI/litellm/pull/19685)
- **Early-exit guards in `completion_cost`** for unused features - [PR #20020](https://github.com/BerriAI/litellm/pull/20020)
- **Optimize `get_litellm_params`** with sparse kwargs extraction - [PR #19884](https://github.com/BerriAI/litellm/pull/19884)
- **Guard debug log f-strings** and remove redundant dict copies - [PR #19961](https://github.com/BerriAI/litellm/pull/19961)
- **Replace enum construction with frozenset lookup** - [PR #20302](https://github.com/BerriAI/litellm/pull/20302)
- **Guard debug f-string in `update_environment_variables`** - [PR #20360](https://github.com/BerriAI/litellm/pull/20360)
- **Warn when budget lookup fails** to surface silent caching misses - [PR #20545](https://github.com/BerriAI/litellm/pull/20545)
- **Add INFO-level session reuse logging** per request for better observability - [PR #20597](https://github.com/BerriAI/litellm/pull/20597)
---
## Database Changes
### Schema Updates
| Table | Change Type | Description | PR | Migration |
| ----- | ----------- | ----------- | -- | --------- |
| `LiteLLM_TeamTable` | New Column | Added `allow_team_guardrail_config` boolean field for team-based guardrail isolation | [PR #20318](https://github.com/BerriAI/litellm/pull/20318) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260205091235_allow_team_guardrail_config/migration.sql) |
| `LiteLLM_DeletedTeamTable` | New Column | Added `allow_team_guardrail_config` boolean field | [PR #20318](https://github.com/BerriAI/litellm/pull/20318) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260205091235_allow_team_guardrail_config/migration.sql) |
| `LiteLLM_TeamTable` | New Column | Added `soft_budget` (double precision) for soft budget alerting | [PR #20530](https://github.com/BerriAI/litellm/pull/20530) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260205144610_add_soft_budget_to_team_table/migration.sql) |
| `LiteLLM_DeletedTeamTable` | New Column | Added `soft_budget` (double precision) | [PR #20653](https://github.com/BerriAI/litellm/pull/20653) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260207110613_add_soft_budget_to_deleted_teams_table/migration.sql) |
| `LiteLLM_MCPServerTable` | New Column | Added `available_on_public_internet` boolean for MCP IP-based access control | [PR #20607](https://github.com/BerriAI/litellm/pull/20607) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260207093506_add_available_on_public_internet_to_mcp_servers/migration.sql) |
---
## Documentation Updates (14 updates)
- Add FAQ for setting up and verifying LITELLM_LICENSE - [PR #20284](https://github.com/BerriAI/litellm/pull/20284)
- Model request tags documentation - [PR #20290](https://github.com/BerriAI/litellm/pull/20290)
- Add Prisma migration troubleshooting guide - [PR #20300](https://github.com/BerriAI/litellm/pull/20300)
- MCP Semantic Filtering documentation - [PR #20316](https://github.com/BerriAI/litellm/pull/20316)
- Add CopilotKit SDK doc as supported agents SDK - [PR #20396](https://github.com/BerriAI/litellm/pull/20396)
- Add documentation for Nova Sonic - [PR #20320](https://github.com/BerriAI/litellm/pull/20320)
- Update Vertex AI Text to Speech doc to show use of audio - [PR #20255](https://github.com/BerriAI/litellm/pull/20255)
- Improve Okta SSO setup guide with step-by-step instructions - [PR #20353](https://github.com/BerriAI/litellm/pull/20353)
- Langfuse doc update - [PR #20443](https://github.com/BerriAI/litellm/pull/20443)
- Expose MCPs on public internet documentation - [PR #20626](https://github.com/BerriAI/litellm/pull/20626)
- Add blog post: Achieving Sub-Millisecond Proxy Overhead - [PR #20309](https://github.com/BerriAI/litellm/pull/20309)
- Add blog post about litellm-observatory - [PR #20622](https://github.com/BerriAI/litellm/pull/20622)
- Update Opus 4.6 blog with adaptive thinking - [PR #20637](https://github.com/BerriAI/litellm/pull/20637)
- `gpt-5-search-api` docs clarifications - [PR #20512](https://github.com/BerriAI/litellm/pull/20512)
---
## New Contributors
* @Quentin-M made their first contribution in [PR #19818](https://github.com/BerriAI/litellm/pull/19818)
* @amirzaushnizer made their first contribution in [PR #20235](https://github.com/BerriAI/litellm/pull/20235)
* @cscguochang made their first contribution in [PR #20214](https://github.com/BerriAI/litellm/pull/20214)
* @krauckbot made their first contribution in [PR #20273](https://github.com/BerriAI/litellm/pull/20273)
* @agrattan0820 made their first contribution in [PR #19784](https://github.com/BerriAI/litellm/pull/19784)
* @nina-hu made their first contribution in [PR #20472](https://github.com/BerriAI/litellm/pull/20472)
* @swayambhu94 made their first contribution in [PR #20469](https://github.com/BerriAI/litellm/pull/20469)
* @ssadedin made their first contribution in [PR #20566](https://github.com/BerriAI/litellm/pull/20566)
---
## Full Changelog
[v1.81.6-nightly...v1.81.9](https://github.com/BerriAI/litellm/compare/v1.81.6-nightly...v1.81.9)

154
docs/my-website/sidebars.js
View file

@ -42,49 +42,62 @@ const sidebars = {
label: "Guardrails",
items: [
"proxy/guardrails/quick_start",
"proxy/guardrails/guardrail_policies",
"proxy/guardrails/guardrail_load_balancing",
"proxy/guardrails/test_playground",
"proxy/guardrails/litellm_content_filter",
{
type: "category",
"label": "Contributing to Guardrails",
label: "Providers",
items: [
...[
"proxy/guardrails/qualifire",
"proxy/guardrails/aim_security",
"proxy/guardrails/onyx_security",
"proxy/guardrails/aporia_api",
"proxy/guardrails/azure_content_guardrail",
"proxy/guardrails/bedrock",
"proxy/guardrails/enkryptai",
"proxy/guardrails/ibm_guardrails",
"proxy/guardrails/grayswan",
"proxy/guardrails/hiddenlayer",
"proxy/guardrails/lasso_security",
"proxy/guardrails/guardrails_ai",
"proxy/guardrails/lakera_ai",
"proxy/guardrails/model_armor",
"proxy/guardrails/noma_security",
"proxy/guardrails/dynamoai",
"proxy/guardrails/openai_moderation",
"proxy/guardrails/pangea",
"proxy/guardrails/pillar_security",
"proxy/guardrails/pii_masking_v2",
"proxy/guardrails/panw_prisma_airs",
"proxy/guardrails/secret_detection",
"proxy/guardrails/custom_guardrail",
"proxy/guardrails/custom_code_guardrail",
"proxy/guardrails/prompt_injection",
"proxy/guardrails/tool_permission",
"proxy/guardrails/zscaler_ai_guard",
"proxy/guardrails/javelin"
].sort(),
],
},
{
type: "category",
label: "Contributing to Guardrails",
items: [
"adding_provider/generic_guardrail_api",
"adding_provider/simple_guardrail_tutorial",
"adding_provider/adding_guardrail_support",
]
},
"proxy/guardrails/test_playground",
"proxy/guardrails/litellm_content_filter",
...[
"proxy/guardrails/qualifire",
"proxy/guardrails/aim_security",
"proxy/guardrails/onyx_security",
"proxy/guardrails/aporia_api",
"proxy/guardrails/azure_content_guardrail",
"proxy/guardrails/bedrock",
"proxy/guardrails/enkryptai",
"proxy/guardrails/ibm_guardrails",
"proxy/guardrails/grayswan",
"proxy/guardrails/hiddenlayer",
"proxy/guardrails/lasso_security",
"proxy/guardrails/guardrails_ai",
"proxy/guardrails/lakera_ai",
"proxy/guardrails/model_armor",
"proxy/guardrails/noma_security",
"proxy/guardrails/dynamoai",
"proxy/guardrails/openai_moderation",
"proxy/guardrails/pangea",
"proxy/guardrails/pillar_security",
"proxy/guardrails/pii_masking_v2",
"proxy/guardrails/panw_prisma_airs",
"proxy/guardrails/secret_detection",
"proxy/guardrails/custom_guardrail",
"proxy/guardrails/custom_code_guardrail",
"proxy/guardrails/prompt_injection",
"proxy/guardrails/tool_permission",
"proxy/guardrails/zscaler_ai_guard",
"proxy/guardrails/javelin"
].sort(),
],
},
{
type: "category",
label: "Policies",
items: [
"proxy/guardrails/guardrail_policies",
"proxy/guardrails/policy_tags",
],
},
{
@ -96,6 +109,11 @@ const sidebars = {
"proxy/prometheus"
]
},
{
type: "doc",
id: "integrations/websearch_interception",
label: "Web Search Integration"
},
{
type: "category",
label: "[Beta] Prompt Management",
@ -125,10 +143,12 @@ const sidebars = {
"tutorials/claude_responses_api",
"tutorials/claude_code_max_subscription",
"tutorials/claude_code_customer_tracking",
"tutorials/claude_code_prompt_cache_routing",
"tutorials/claude_code_websearch",
"tutorials/claude_mcp",
"tutorials/claude_non_anthropic_models",
"tutorials/claude_code_plugin_marketplace",
"tutorials/claude_code_beta_headers",
]
},
"tutorials/opencode_integration",
@ -222,6 +242,7 @@ const sidebars = {
label: "Configuration",
items: [
"set_keys",
"proxy_auth",
"caching/all_caches",
],
},
@ -286,40 +307,52 @@ const sidebars = {
label: "Admin UI",
items: [
"proxy/ui",
"proxy/admin_ui_sso",
"proxy/custom_root_ui",
"proxy/custom_sso",
"proxy/ai_hub",
"proxy/model_compare_ui",
"proxy/ui_credentials",
"tutorials/scim_litellm",
{
type: "category",
label: "UI User/Team Management",
label: "Setup & SSO",
items: [
"proxy/access_control",
"proxy/public_teams",
"proxy/admin_ui_sso",
"proxy/custom_sso",
"proxy/custom_root_ui",
"tutorials/scim_litellm",
]
},
{
type: "category",
label: "Models",
items: [
"proxy/ui_credentials",
"proxy/ai_hub",
"proxy/model_compare_ui",
]
},
{
type: "category",
label: "Teams & Organizations",
items: [
"proxy/access_control",
"proxy/self_serve",
"proxy/public_teams",
"proxy/ui/bulk_edit_users",
"proxy/ui/page_visibility",
]
},
{
type: "category",
label: "UI Usage Tracking",
label: "Observability: Usage",
items: [
"proxy/customer_usage",
"proxy/endpoint_activity"
"proxy/endpoint_activity",
]
},
{
type: "category",
label: "UI Logs",
label: "Logs",
items: [
"proxy/ui_logs",
"proxy/ui_spend_log_settings",
"proxy/ui_logs_sessions",
"proxy/deleted_keys_teams"
"proxy/deleted_keys_teams",
]
}
],
@ -367,6 +400,7 @@ const sidebars = {
items: [
"proxy/users",
"proxy/team_budgets",
"proxy/ui_team_soft_budget_alerts",
"proxy/tag_budgets",
"proxy/customers",
"proxy/dynamic_rate_limit",
@ -375,6 +409,16 @@ const sidebars = {
],
},
"proxy/caching",
{
type: "link",
label: "Guardrails",
href: "https://docs.litellm.ai/docs/proxy/guardrails/quick_start",
},
{
type: "link",
label: "Policies",
href: "https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies",
},
{
type: "category",
label: "Create Custom Plugins",
@ -542,6 +586,8 @@ const sidebars = {
items: [
"mcp",
"mcp_usage",
"mcp_oauth",
"mcp_public_internet",
"mcp_semantic_filter",
"mcp_control",
"mcp_cost",
@ -1059,6 +1105,18 @@ const sidebars = {
"troubleshoot/cpu_issues",
"troubleshoot/memory_issues",
"troubleshoot/spend_queue_warnings",
"troubleshoot/max_callbacks",
],
},
{
type: "category",
label: "Blog",
items: [
{
type: "link",
label: "Incident: Broken Model Cost Map",
href: "/blog/model-cost-map-incident",
},
],
},
],

133
docs/my-website/src/components/MiddlewareDiagrams/BaseHTTPMiddlewareAnimation.tsx Normal file
View file

@ -0,0 +1,133 @@
import React, { useState, useEffect, useCallback, useRef } from 'react';
import styles from './styles.module.css';
interface Stage {
label: string;
subtitle: string;
code: string;
}
const STAGES: Stage[] = [
{
label: 'Request Wrapping',
subtitle: '_CachedRequest',
code: 'request = _CachedRequest(scope, receive)',
},
{
label: 'Sync Event',
subtitle: 'anyio.Event()',
code: 'response_sent = anyio.Event()',
},
{
label: 'Memory Stream',
subtitle: 'create_memory_object_stream()',
code: 'send_stream, recv_stream = anyio.create_memory_object_stream()',
},
{
label: 'Task Group',
subtitle: 'create_task_group()',
code: 'async with anyio.create_task_group() as task_group:',
},
{
label: 'Background Task',
subtitle: 'task_group.start_soon(coro)',
code: 'task_group.start_soon(coro) # app runs in separate task',
},
{
label: 'Nested Task Group',
subtitle: 'receive_or_disconnect()',
code: 'async with anyio.create_task_group() as task_group: ...',
},
{
label: 'Response Wrapping',
subtitle: '_StreamingResponse',
code: 'response = _StreamingResponse(status_code=..., content=body_stream())',
},
];
const INTERVAL_MS = 1200;
const PAUSE_MS = 600;
export default function BaseHTTPMiddlewareAnimation() {
const [activeStage, setActiveStage] = useState(0);
const [paused, setPaused] = useState(false);
const [expandedStage, setExpandedStage] = useState<number | null>(null);
const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const clearTimer = useCallback(() => {
if (timerRef.current !== null) {
clearTimeout(timerRef.current);
timerRef.current = null;
}
}, []);
useEffect(() => {
if (paused) return;
const advance = () => {
setActiveStage((prev) => {
const next = (prev + 1) % STAGES.length;
// If wrapping around, add extra pause
if (next === 0) {
timerRef.current = setTimeout(() => {
timerRef.current = setTimeout(advance, INTERVAL_MS);
}, PAUSE_MS);
return next;
}
timerRef.current = setTimeout(advance, INTERVAL_MS);
return next;
});
};
timerRef.current = setTimeout(advance, INTERVAL_MS);
return clearTimer;
}, [paused, clearTimer]);
const handleStageClick = (index: number) => {
clearTimer();
setPaused(true);
setActiveStage(index);
if (expandedStage === index) {
// Close panel and resume
setExpandedStage(null);
setPaused(false);
} else {
setExpandedStage(index);
}
};
return (
<div className={styles.pipelineWrapper}>
<div className={styles.pipelineLabel}>7 steps per request</div>
<div className={styles.pipeline}>
{STAGES.map((stage, i) => (
<div className={styles.stageWrapper} key={i}>
<div
className={`${styles.stage} ${activeStage === i ? styles.stageActive : ''}`}
onClick={() => handleStageClick(i)}
role="button"
tabIndex={0}
onKeyDown={(e) => {
if (e.key === 'Enter' || e.key === ' ') handleStageClick(i);
}}
>
<div className={styles.stageNumber}>{i + 1}</div>
<div className={styles.stageLabel}>{stage.label}</div>
<div className={styles.stageSubtitle}>{stage.subtitle}</div>
</div>
</div>
))}
</div>
<div
className={`${styles.codePanel} ${expandedStage !== null ? styles.codePanelOpen : ''}`}
>
{expandedStage !== null && (
<pre className={styles.codePanelCode}>
<code>{STAGES[expandedStage].code}</code>
</pre>
)}
</div>
</div>
);
}

337
docs/my-website/src/components/MiddlewareDiagrams/BenchmarkVisualization.tsx Normal file
View file

@ -0,0 +1,337 @@
import React, { useState, useEffect, useRef, useCallback } from 'react';
import styles from './styles.module.css';
/* ── Constants ── */
const TOTAL_REQUESTS = 50_000;
const DURATION_AFTER_MS = 8_000; // "After" column finishes in 8s
const DURATION_BEFORE_MS = 13_920; // 74% slower → 8000 * 1.74
const TICK_MS = 50;
const RESET_PAUSE_MS = 2_000;
const MAX_DOTS = 14;
const BEFORE_RPS = 3_785;
const AFTER_RPS = 6_577;
const BEFORE_P50 = 21;
const AFTER_P50 = 13;
const BEFORE_LAYERS = [
{ label: 'ab client', warning: false },
{ label: 'uvicorn \u00B7 1 worker', warning: false },
{ label: 'ASGI Middleware', warning: false },
{ label: 'BaseHTTPMiddleware', warning: true },
{ label: 'GET /health \u2192 "ok"', warning: false },
];
const AFTER_LAYERS = [
{ label: 'ab client', warning: false },
{ label: 'uvicorn \u00B7 1 worker', warning: false },
{ label: 'ASGI Middleware', warning: false },
{ label: 'ASGI Middleware', warning: false },
{ label: 'GET /health \u2192 "ok"', warning: false },
];
const BENCHMARK_RUNS = [
{ config: 'Before (1 ASGI + 1 BaseHTTP)', run: 1, rps: 3596, p50: 21 },
{ config: 'Before (1 ASGI + 1 BaseHTTP)', run: 2, rps: 3599, p50: 21 },
{ config: 'Before (1 ASGI + 1 BaseHTTP)', run: 3, rps: 4161, p50: 21 },
{ config: 'After (2x Pure ASGI)', run: 1, rps: 6504, p50: 13 },
{ config: 'After (2x Pure ASGI)', run: 2, rps: 6631, p50: 13 },
{ config: 'After (2x Pure ASGI)', run: 3, rps: 6595, p50: 13 },
];
/* ── Dot type ── */
interface Dot {
id: number;
progress: number; // 0..1 (top to bottom)
}
/* ── Component ── */
export default function BenchmarkVisualization() {
const [elapsed, setElapsed] = useState(0);
const [running, setRunning] = useState(false);
const [afterDone, setAfterDone] = useState(false);
const [beforeDone, setBeforeDone] = useState(false);
const [tableOpen, setTableOpen] = useState(false);
const [beforeDots, setBeforeDots] = useState<Dot[]>([]);
const [afterDots, setAfterDots] = useState<Dot[]>([]);
const dotIdRef = useRef(0);
const observerRef = useRef<IntersectionObserver | null>(null);
const wrapperRef = useRef<HTMLDivElement | null>(null);
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
const hasStartedRef = useRef(false);
const beforeProgress = Math.min(elapsed / DURATION_BEFORE_MS, 1);
const afterProgress = Math.min(elapsed / DURATION_AFTER_MS, 1);
const beforeCompleted = Math.round(beforeProgress * TOTAL_REQUESTS);
const afterCompleted = Math.round(afterProgress * TOTAL_REQUESTS);
const beforeCurrentRPS = running && !beforeDone
? Math.round(BEFORE_RPS * (0.9 + Math.random() * 0.2))
: beforeDone ? 0 : 0;
const afterCurrentRPS = running && !afterDone
? Math.round(AFTER_RPS * (0.9 + Math.random() * 0.2))
: afterDone ? 0 : 0;
const reset = useCallback(() => {
setElapsed(0);
setAfterDone(false);
setBeforeDone(false);
setBeforeDots([]);
setAfterDots([]);
dotIdRef.current = 0;
}, []);
// Start/restart loop
const startSimulation = useCallback(() => {
reset();
setRunning(true);
}, [reset]);
// IntersectionObserver to auto-start on scroll
useEffect(() => {
observerRef.current = new IntersectionObserver(
([entry]) => {
if (entry.isIntersecting && !hasStartedRef.current) {
hasStartedRef.current = true;
startSimulation();
}
},
{ threshold: 0.3 }
);
if (wrapperRef.current) {
observerRef.current.observe(wrapperRef.current);
}
return () => {
observerRef.current?.disconnect();
};
}, [startSimulation]);
// Main tick
useEffect(() => {
if (!running) return;
timerRef.current = setInterval(() => {
setElapsed((prev) => {
const next = prev + TICK_MS;
if (next >= DURATION_AFTER_MS) setAfterDone(true);
if (next >= DURATION_BEFORE_MS) setBeforeDone(true);
// Both done → schedule reset
if (next >= DURATION_BEFORE_MS) {
setTimeout(() => {
startSimulation();
}, RESET_PAUSE_MS);
setRunning(false);
return next;
}
return next;
});
}, TICK_MS);
return () => {
if (timerRef.current) clearInterval(timerRef.current);
};
}, [running, startSimulation]);
// Dot animation
useEffect(() => {
if (!running) return;
const dotInterval = setInterval(() => {
const spawnBefore = !beforeDone && Math.random() < 0.4;
const spawnAfter = !afterDone && Math.random() < 0.65;
if (spawnBefore) {
setBeforeDots((prev) => {
const dots = [...prev, { id: dotIdRef.current++, progress: 0 }];
return dots.slice(-MAX_DOTS);
});
}
if (spawnAfter) {
setAfterDots((prev) => {
const dots = [...prev, { id: dotIdRef.current++, progress: 0 }];
return dots.slice(-MAX_DOTS);
});
}
// Advance existing dots
setBeforeDots((prev) =>
prev
.map((d) => ({ ...d, progress: d.progress + 0.08 }))
.filter((d) => d.progress <= 1)
);
setAfterDots((prev) =>
prev
.map((d) => ({ ...d, progress: d.progress + 0.14 }))
.filter((d) => d.progress <= 1)
);
}, 100);
return () => clearInterval(dotInterval);
}, [running, beforeDone, afterDone]);
const renderFlowStack = (
layers: { label: string; warning: boolean }[],
dots: Dot[],
isBefore: boolean
) => (
<div className={styles.flowStack}>
<div className={styles.dotsCanvas}>
{dots.map((dot) => (
<div
key={dot.id}
className={`${styles.dot} ${isBefore ? styles.dotSlow : styles.dotFast}`}
style={{
top: `${dot.progress * 92}%`,
left: `${48 + Math.sin(dot.id * 1.7) * 12}%`,
opacity: dot.progress > 0.85 ? (1 - dot.progress) * 6 : 0.8,
}}
/>
))}
</div>
{layers.map((layer, i) => (
<React.Fragment key={i}>
{i > 0 && <div className={styles.flowArrow}>&darr;</div>}
<div
className={`${styles.flowLayer} ${layer.warning ? styles.flowLayerWarning : ''}`}
>
{layer.label}
{layer.warning && <span className={styles.overheadTag}>&larr; overhead</span>}
</div>
</React.Fragment>
))}
</div>
);
const formatNum = (n: number) => n.toLocaleString();
return (
<div className={styles.benchmarkWrapper} ref={wrapperRef}>
<div className={styles.benchmarkConfig}>
50,000 requests &middot; 1,000 concurrent &middot; 1 worker
</div>
<div className={styles.benchmarkColumns}>
{/* Before column */}
<div className={styles.benchmarkColumn}>
<div className={`${styles.columnTitle} ${styles.columnTitleBefore}`}>
Before (1 ASGI + 1 BaseHTTP)
{beforeDone && (
<span className={`${styles.doneBadge} ${styles.doneBadgeBefore}`}>done</span>
)}
</div>
{renderFlowStack(BEFORE_LAYERS, beforeDots, true)}
<div className={styles.statsRow}>
<div className={styles.stat}>
<div className={styles.statValue}>{formatNum(beforeCurrentRPS)}</div>
<div className={styles.statLabel}>RPS</div>
</div>
<div className={styles.stat}>
<div className={styles.statValue}>{formatNum(beforeCompleted)}</div>
<div className={styles.statLabel}>Completed</div>
</div>
<div className={styles.stat}>
<div className={styles.statValue}>{BEFORE_P50}ms</div>
<div className={styles.statLabel}>P50</div>
</div>
</div>
<div className={styles.progressBar}>
<div
className={`${styles.progressFill} ${styles.progressFillBefore}`}
style={{ width: `${beforeProgress * 100}%` }}
/>
</div>
</div>
{/* After column */}
<div className={styles.benchmarkColumn}>
<div className={`${styles.columnTitle} ${styles.columnTitleAfter}`}>
After (2x Pure ASGI)
{afterDone && (
<span className={`${styles.doneBadge} ${styles.doneBadgeAfter}`}>done</span>
)}
</div>
{renderFlowStack(AFTER_LAYERS, afterDots, false)}
<div className={styles.statsRow}>
<div className={styles.stat}>
<div className={styles.statValue}>{formatNum(afterCurrentRPS)}</div>
<div className={styles.statLabel}>RPS</div>
</div>
<div className={styles.stat}>
<div className={styles.statValue}>{formatNum(afterCompleted)}</div>
<div className={styles.statLabel}>Completed</div>
</div>
<div className={styles.stat}>
<div className={styles.statValue}>{AFTER_P50}ms</div>
<div className={styles.statLabel}>P50</div>
</div>
</div>
<div className={styles.progressBar}>
<div
className={`${styles.progressFill} ${styles.progressFillAfter}`}
style={{ width: `${afterProgress * 100}%` }}
/>
</div>
</div>
</div>
{/* Summary stats */}
<div className={styles.summaryStats}>
<div className={styles.summaryItem}>
<div className={styles.summaryValue}>+74%</div>
<div className={styles.summaryLabel}>Throughput (RPS)</div>
</div>
<div className={styles.summaryItem}>
<div className={styles.summaryValue}>-38%</div>
<div className={styles.summaryLabel}>Median Latency (P50)</div>
</div>
</div>
{/* Collapsible per-run data */}
<div className={styles.collapsible}>
<button
className={styles.collapsibleToggle}
onClick={() => setTableOpen(!tableOpen)}
>
<span
className={`${styles.collapsibleChevron} ${
tableOpen ? styles.collapsibleChevronOpen : ''
}`}
>
&#9654;
</span>
Per-run data (3 runs each)
</button>
<div
className={`${styles.collapsibleContent} ${
tableOpen ? styles.collapsibleContentOpen : ''
}`}
>
<table className={styles.dataTable}>
<thead>
<tr>
<th>Config</th>
<th>Run</th>
<th>RPS</th>
<th>P50 (ms)</th>
</tr>
</thead>
<tbody>
{BENCHMARK_RUNS.map((row, i) => (
<tr key={i}>
<td>{row.config}</td>
<td>{row.run}</td>
<td>{formatNum(row.rps)}</td>
<td>{row.p50}</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
</div>
);
}

67
docs/my-website/src/components/MiddlewareDiagrams/PureASGIAnimation.tsx Normal file
View file

@ -0,0 +1,67 @@
import React, { useState, useEffect, useRef, useCallback } from 'react';
import styles from './styles.module.css';
interface Stage {
label: string;
subtitle: string;
}
const STAGES: Stage[] = [
{ label: 'Scope Check', subtitle: 'scope["type"] != "http"' },
{ label: 'Direct Call', subtitle: 'await self.app(scope, receive, send)' },
];
const INTERVAL_MS = 1200;
const PAUSE_MS = 600;
export default function PureASGIAnimation() {
const [activeStage, setActiveStage] = useState(0);
const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const clearTimer = useCallback(() => {
if (timerRef.current !== null) {
clearTimeout(timerRef.current);
timerRef.current = null;
}
}, []);
useEffect(() => {
const advance = () => {
setActiveStage((prev) => {
const next = (prev + 1) % STAGES.length;
if (next === 0) {
timerRef.current = setTimeout(() => {
timerRef.current = setTimeout(advance, INTERVAL_MS);
}, PAUSE_MS);
return next;
}
timerRef.current = setTimeout(advance, INTERVAL_MS);
return next;
});
};
timerRef.current = setTimeout(advance, INTERVAL_MS);
return clearTimer;
}, [clearTimer]);
return (
<div className={styles.pipelineWrapper}>
<div className={styles.pipelineLabel}>2 steps per request</div>
<div className={`${styles.pipeline} ${styles.pipelineTwoCol}`}>
{STAGES.map((stage, i) => (
<div className={styles.stageWrapper} key={i}>
<div
className={`${styles.stage} ${styles.stageNoClick} ${
activeStage === i ? styles.stageActiveGreen : ''
}`}
>
<div className={styles.stageNumber}>{i + 1}</div>
<div className={styles.stageLabel}>{stage.label}</div>
<div className={styles.stageSubtitle}>{stage.subtitle}</div>
</div>
</div>
))}
</div>
</div>
);
}

3
docs/my-website/src/components/MiddlewareDiagrams/index.tsx Normal file
View file

@ -0,0 +1,3 @@
export { default as BaseHTTPMiddlewareAnimation } from './BaseHTTPMiddlewareAnimation';
export { default as PureASGIAnimation } from './PureASGIAnimation';
export { default as BenchmarkVisualization } from './BenchmarkVisualization';

494
docs/my-website/src/components/MiddlewareDiagrams/styles.module.css Normal file
View file

@ -0,0 +1,494 @@
/* ── Shared custom properties ── */
:root {
--mw-stage-bg: #f8f9fa;
--mw-stage-border: #dee2e6;
--mw-stage-active-bg: #e8f4fd;
--mw-stage-active-border: #3b82f6;
--mw-stage-green-active-bg: #ecfdf5;
--mw-stage-green-active-border: #10b981;
--mw-dot-color: #3b82f6;
--mw-warning-accent: #ef4444;
--mw-success-accent: #10b981;
--mw-text-primary: #1a1a2e;
--mw-text-secondary: #6b7280;
--mw-code-bg: #f1f5f9;
--mw-panel-bg: #ffffff;
--mw-panel-border: #e5e7eb;
--mw-bar-bg: #e5e7eb;
--mw-arrow-color: #9ca3af;
--mw-column-bg: #fafafa;
--mw-column-border: #e5e7eb;
--mw-layer-bg: #f3f4f6;
--mw-layer-border: #d1d5db;
--mw-layer-warning-bg: #fef2f2;
--mw-layer-warning-border: #fca5a5;
--mw-progress-bg: #e5e7eb;
}
[data-theme='dark'] {
--mw-stage-bg: #1e1e2e;
--mw-stage-border: #374151;
--mw-stage-active-bg: #1e3a5f;
--mw-stage-active-border: #60a5fa;
--mw-stage-green-active-bg: #064e3b;
--mw-stage-green-active-border: #34d399;
--mw-dot-color: #60a5fa;
--mw-warning-accent: #f87171;
--mw-success-accent: #34d399;
--mw-text-primary: #e5e7eb;
--mw-text-secondary: #9ca3af;
--mw-code-bg: #1e293b;
--mw-panel-bg: #111827;
--mw-panel-border: #374151;
--mw-bar-bg: #374151;
--mw-arrow-color: #6b7280;
--mw-column-bg: #111827;
--mw-column-border: #374151;
--mw-layer-bg: #1f2937;
--mw-layer-border: #4b5563;
--mw-layer-warning-bg: #451a1a;
--mw-layer-warning-border: #b91c1c;
--mw-progress-bg: #374151;
}
/* ── Pipeline (shared between BaseHTTP and PureASGI) ── */
.pipelineWrapper {
margin: 1.5rem 0;
}
.pipelineLabel {
text-align: center;
font-size: 0.85rem;
font-weight: 600;
color: var(--mw-text-secondary);
margin-bottom: 0.75rem;
text-transform: uppercase;
letter-spacing: 0.05em;
}
.pipeline {
display: flex;
flex-wrap: wrap;
justify-content: center;
align-items: stretch;
gap: 0.75rem;
padding: 0.5rem 0;
}
.pipelineTwoCol {
max-width: 480px;
margin: 0 auto;
}
.stageWrapper {
display: flex;
align-items: center;
width: 160px;
flex-shrink: 0;
}
.pipelineTwoCol .stageWrapper {
width: 200px;
}
.arrow {
display: none;
}
.stage {
flex: 1;
padding: 0.85rem 0.75rem;
min-height: 100px;
display: flex;
flex-direction: column;
justify-content: center;
background: var(--mw-stage-bg);
border: 2px solid var(--mw-stage-border);
border-radius: 8px;
text-align: center;
cursor: pointer;
transition: background 0.4s ease, border-color 0.4s ease, box-shadow 0.4s ease;
user-select: none;
}
.stage:hover {
border-color: var(--mw-stage-active-border);
}
.stageActive {
background: var(--mw-stage-active-bg);
border-color: var(--mw-stage-active-border);
box-shadow: 0 0 0 3px rgba(59, 130, 246, 0.15);
}
.stageActiveGreen {
background: var(--mw-stage-green-active-bg);
border-color: var(--mw-stage-green-active-border);
box-shadow: 0 0 0 3px rgba(16, 185, 129, 0.15);
}
.stageNoClick {
cursor: default;
}
.stageNumber {
font-size: 0.7rem;
font-weight: 700;
color: var(--mw-text-secondary);
margin-bottom: 0.3rem;
}
.stageLabel {
font-size: 0.85rem;
font-weight: 600;
color: var(--mw-text-primary);
margin-bottom: 0.25rem;
line-height: 1.3;
}
.stageSubtitle {
font-size: 0.72rem;
color: var(--mw-text-secondary);
font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
word-break: break-word;
line-height: 1.3;
}
/* ── Code panel (accordion) ── */
.codePanel {
max-height: 0;
overflow: hidden;
transition: max-height 0.35s ease, padding 0.35s ease;
background: var(--mw-code-bg);
border-radius: 0 0 8px 8px;
margin-top: 0.5rem;
}
.codePanelOpen {
max-height: 120px;
padding: 0.75rem 1rem;
}
.codePanelCode {
font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
font-size: 0.8rem;
color: var(--mw-text-primary);
white-space: pre;
margin: 0;
line-height: 1.5;
}
/* ── Benchmark Visualization ── */
.benchmarkWrapper {
margin: 1.5rem 0;
}
.benchmarkConfig {
text-align: center;
font-size: 0.85rem;
color: var(--mw-text-secondary);
margin-bottom: 1rem;
font-weight: 500;
}
.benchmarkColumns {
display: flex;
gap: 1.5rem;
}
.benchmarkColumn {
flex: 1;
background: var(--mw-column-bg);
border: 1px solid var(--mw-column-border);
border-radius: 12px;
padding: 1.25rem;
position: relative;
overflow: hidden;
}
.columnTitle {
font-size: 0.9rem;
font-weight: 700;
color: var(--mw-text-primary);
text-align: center;
margin-bottom: 1rem;
}
.columnTitleBefore {
color: var(--mw-warning-accent);
}
.columnTitleAfter {
color: var(--mw-success-accent);
}
/* ── Request flow stack ── */
.flowStack {
display: flex;
flex-direction: column;
align-items: center;
gap: 0;
position: relative;
min-height: 280px;
}
.flowLayer {
width: 100%;
max-width: 260px;
padding: 0.6rem 0.75rem;
background: var(--mw-layer-bg);
border: 1px solid var(--mw-layer-border);
border-radius: 6px;
text-align: center;
font-size: 0.78rem;
font-weight: 500;
color: var(--mw-text-primary);
position: relative;
z-index: 1;
}
.flowLayerWarning {
background: var(--mw-layer-warning-bg);
border-color: var(--mw-layer-warning-border);
font-weight: 700;
}
.flowArrow {
display: flex;
justify-content: center;
color: var(--mw-arrow-color);
font-size: 0.9rem;
padding: 0.15rem 0;
position: relative;
z-index: 0;
min-height: 20px;
}
.overheadTag {
font-size: 0.65rem;
color: var(--mw-warning-accent);
margin-left: 0.4rem;
}
/* ── Dots layer (canvas for flowing dots) ── */
.dotsCanvas {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
pointer-events: none;
z-index: 2;
}
.dot {
position: absolute;
width: 6px;
height: 6px;
border-radius: 50%;
background: var(--mw-dot-color);
opacity: 0.8;
}
.dotSlow {
background: var(--mw-warning-accent);
}
.dotFast {
background: var(--mw-success-accent);
}
/* ── Stats & progress ── */
.statsRow {
display: flex;
justify-content: space-around;
margin-top: 1rem;
padding-top: 0.75rem;
border-top: 1px solid var(--mw-panel-border);
}
.stat {
text-align: center;
}
.statValue {
font-size: 1.1rem;
font-weight: 700;
color: var(--mw-text-primary);
font-variant-numeric: tabular-nums;
}
.statLabel {
font-size: 0.7rem;
color: var(--mw-text-secondary);
text-transform: uppercase;
letter-spacing: 0.04em;
}
.progressBar {
width: 100%;
height: 6px;
background: var(--mw-progress-bg);
border-radius: 3px;
margin-top: 0.75rem;
overflow: hidden;
}
.progressFill {
height: 100%;
border-radius: 3px;
transition: width 0.1s linear;
}
.progressFillBefore {
background: var(--mw-warning-accent);
}
.progressFillAfter {
background: var(--mw-success-accent);
}
/* ── Summary stats below simulation ── */
.summaryStats {
display: flex;
justify-content: center;
gap: 2rem;
margin-top: 1.5rem;
flex-wrap: wrap;
}
.summaryItem {
text-align: center;
padding: 0.75rem 1.25rem;
background: var(--mw-stage-bg);
border-radius: 8px;
border: 1px solid var(--mw-panel-border);
}
.summaryValue {
font-size: 1.5rem;
font-weight: 800;
color: var(--mw-success-accent);
}
.summaryLabel {
font-size: 0.8rem;
color: var(--mw-text-secondary);
margin-top: 0.2rem;
}
/* ── Collapsible table ── */
.collapsible {
margin-top: 1.5rem;
}
.collapsibleToggle {
background: none;
border: 1px solid var(--mw-panel-border);
border-radius: 6px;
padding: 0.5rem 1rem;
cursor: pointer;
font-size: 0.85rem;
color: var(--mw-text-primary);
width: 100%;
text-align: left;
display: flex;
align-items: center;
gap: 0.5rem;
transition: background 0.2s;
}
.collapsibleToggle:hover {
background: var(--mw-stage-bg);
}
.collapsibleChevron {
transition: transform 0.3s ease;
font-size: 0.7rem;
}
.collapsibleChevronOpen {
transform: rotate(90deg);
}
.collapsibleContent {
max-height: 0;
overflow: hidden;
transition: max-height 0.35s ease;
}
.collapsibleContentOpen {
max-height: 600px;
}
.dataTable {
width: 100%;
border-collapse: collapse;
margin-top: 0.75rem;
font-size: 0.85rem;
}
.dataTable th,
.dataTable td {
padding: 0.5rem 0.75rem;
text-align: left;
border-bottom: 1px solid var(--mw-panel-border);
}
.dataTable th {
font-weight: 600;
color: var(--mw-text-secondary);
font-size: 0.75rem;
text-transform: uppercase;
letter-spacing: 0.04em;
}
.dataTable td {
color: var(--mw-text-primary);
font-variant-numeric: tabular-nums;
}
/* ── Reproduce section ── */
.reproduceSection {
margin-top: 1rem;
}
/* ── Done badge ── */
.doneBadge {
display: inline-block;
font-size: 0.75rem;
font-weight: 600;
padding: 0.2rem 0.6rem;
border-radius: 4px;
margin-left: 0.5rem;
}
.doneBadgeBefore {
color: var(--mw-warning-accent);
background: var(--mw-layer-warning-bg);
}
.doneBadgeAfter {
color: var(--mw-success-accent);
background: var(--mw-stage-green-active-bg);
}
/* ── Responsive ── */
@media (max-width: 768px) {
.stageWrapper {
width: 140px;
}
.pipelineTwoCol .stageWrapper {
width: 160px;
}
.benchmarkColumns {
flex-direction: column;
}
.summaryStats {
flex-direction: column;
align-items: center;
}
}

BIN
enterprise/dist/litellm_enterprise-0.1.30-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.30.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.31-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.31.tar.gz vendored Normal file
View file

Binary file not shown.

119
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py
View file

@ -30,8 +30,15 @@ from litellm.integrations.email_templates.user_invitation_email import (
from litellm.integrations.email_templates.templates import (
MAX_BUDGET_ALERT_EMAIL_TEMPLATE,
SOFT_BUDGET_ALERT_EMAIL_TEMPLATE,
TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE,
)
from litellm.proxy._types import (
CallInfo,
InvitationNew,
Litellm_EntityType,
UserAPIKeyAuth,
WebhookEvent,
)
from litellm.proxy._types import CallInfo, InvitationNew, UserAPIKeyAuth, WebhookEvent
from litellm.secret_managers.main import get_secret_bool
from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL
from litellm.constants import (
@ -217,6 +224,78 @@ class BaseEmailLogger(CustomLogger):
)
pass
async def send_team_soft_budget_alert_email(self, event: WebhookEvent):
"""
Send email to team members when team soft budget is crossed
Supports multiple recipients via alert_emails field from team metadata
"""
# Collect all recipient emails
recipient_emails: List[str] = []
# Add additional alert emails from team metadata.soft_budget_alert_emails
if hasattr(event, "alert_emails") and event.alert_emails:
for email in event.alert_emails:
if email and email not in recipient_emails: # Avoid duplicates
recipient_emails.append(email)
# If no recipients found, skip sending
if not recipient_emails:
verbose_proxy_logger.warning(
f"No recipient emails found for team soft budget alert. event={event.model_dump(exclude_none=True)}"
)
return
# Validate that we have at least one valid email address
first_recipient_email = recipient_emails[0]
if not first_recipient_email or not first_recipient_email.strip():
verbose_proxy_logger.warning(
f"Invalid recipient email found for team soft budget alert. event={event.model_dump(exclude_none=True)}"
)
return
verbose_proxy_logger.debug(
f"send_team_soft_budget_alert_email_event: {json.dumps(event.model_dump(exclude_none=True), indent=4, default=str)}"
)
# Get email params using the first recipient email (for template formatting)
# For team alerts with alert_emails, we don't need user_id lookup since we already have email addresses
# Pass user_id=None to prevent _get_email_params from trying to look up email from a potentially None user_id
email_params = await self._get_email_params(
email_event=EmailEvent.soft_budget_crossed,
user_id=None, # Team alerts don't require user_id when alert_emails are provided
user_email=first_recipient_email,
event_message=event.event_message,
)
# Format budget values
soft_budget_str = f"${event.soft_budget}" if event.soft_budget is not None else "N/A"
spend_str = f"${event.spend}" if event.spend is not None else "$0.00"
max_budget_info = ""
if event.max_budget is not None:
max_budget_info = f"<b>Maximum Budget:</b> ${event.max_budget} <br />"
# Use team alias or generic greeting
team_alias = event.team_alias or "Team"
email_html_content = TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE.format(
email_logo_url=email_params.logo_url,
team_alias=team_alias,
soft_budget=soft_budget_str,
spend=spend_str,
max_budget_info=max_budget_info,
base_url=email_params.base_url,
email_support_contact=email_params.support_contact,
)
# Send email to all recipients
await self.send_email(
from_email=self.DEFAULT_LITELLM_EMAIL,
to_email=recipient_emails,
subject=email_params.subject,
html_body=email_html_content,
)
pass
async def send_max_budget_alert_email(self, event: WebhookEvent):
"""
Send email to user when max budget alert threshold is reached
@ -285,15 +364,36 @@ class BaseEmailLogger(CustomLogger):
# - Don't re-alert, if alert already sent
_cache: DualCache = self.internal_usage_cache
# percent of max_budget left to spend
if user_info.max_budget is None and user_info.soft_budget is None:
return
# For soft_budget alerts, check if we've already sent an alert
if type == "soft_budget":
# For team soft budget alerts, we only need team soft_budget to be set
# For other entity types, we need either max_budget or soft_budget
if user_info.event_group == Litellm_EntityType.TEAM:
if user_info.soft_budget is None:
return
# For team soft budget alerts, require alert_emails to be configured
# Team soft budget alerts are sent via metadata.soft_budget_alerting_emails
if user_info.alert_emails is None or len(user_info.alert_emails) == 0:
verbose_proxy_logger.debug(
"Skipping team soft budget email alert: no alert_emails configured",
)
return
else:
# For non-team alerts, require either max_budget or soft_budget
if user_info.max_budget is None and user_info.soft_budget is None:
return
if user_info.soft_budget is not None and user_info.spend >= user_info.soft_budget:
# Generate cache key based on event type and identifier
_id = user_info.token or user_info.user_id or "default_id"
# Use appropriate ID based on event_group to ensure unique cache keys per entity type
if user_info.event_group == Litellm_EntityType.TEAM:
_id = user_info.team_id or "default_id"
elif user_info.event_group == Litellm_EntityType.ORGANIZATION:
_id = user_info.organization_id or "default_id"
elif user_info.event_group == Litellm_EntityType.USER:
_id = user_info.user_id or "default_id"
else:
# For KEY and other types, use token or user_id
_id = user_info.token or user_info.user_id or "default_id"
_cache_key = f"email_budget_alerts:soft_budget_crossed:{_id}"
# Check if we've already sent this alert
@ -318,10 +418,15 @@ class BaseEmailLogger(CustomLogger):
projected_exceeded_date=user_info.projected_exceeded_date,
projected_spend=user_info.projected_spend,
event_group=user_info.event_group,
alert_emails=user_info.alert_emails,
)
try:
await self.send_soft_budget_alert_email(webhook_event)
# Use team-specific function for team alerts, otherwise use standard function
if user_info.event_group == Litellm_EntityType.TEAM:
await self.send_team_soft_budget_alert_email(webhook_event)
else:
await self.send_soft_budget_alert_email(webhook_event)
# Cache the alert to prevent duplicate sends
await _cache.async_set_cache(

98
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -899,49 +899,49 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
batch_id=response.id, model_id=model_id
)
if (
response.output_file_id and model_id
): # return a file id with the model_id and output_file_id
original_output_file_id = response.output_file_id
response.output_file_id = self.get_unified_output_file_id(
output_file_id=response.output_file_id,
model_id=model_id,
model_name=model_name,
)
# Fetch the actual file object for the output file
file_object = None
try:
# Use litellm to retrieve the file object from the provider
from litellm import afile_retrieve
file_object = await afile_retrieve(
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
file_id=original_output_file_id
# Handle both output_file_id and error_file_id
for file_attr in ["output_file_id", "error_file_id"]:
file_id_value = getattr(response, file_attr, None)
if file_id_value and model_id:
original_file_id = file_id_value
unified_file_id = self.get_unified_output_file_id(
output_file_id=original_file_id,
model_id=model_id,
model_name=model_name,
)
verbose_logger.debug(
f"Successfully retrieved file object for output_file_id={original_output_file_id}"
setattr(response, file_attr, unified_file_id)
# Fetch the actual file object from the provider
file_object = None
try:
# Use litellm to retrieve the file object from the provider
from litellm import afile_retrieve
file_object = await afile_retrieve(
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
file_id=original_file_id
)
verbose_logger.debug(
f"Successfully retrieved file object for {file_attr}={original_file_id}"
)
except Exception as e:
verbose_logger.warning(
f"Failed to retrieve file object for {file_attr}={original_file_id}: {str(e)}. Storing with None and will fetch on-demand."
)
await self.store_unified_file_id(
file_id=unified_file_id,
file_object=file_object,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_mappings={model_id: original_file_id},
user_api_key_dict=user_api_key_dict,
)
except Exception as e:
verbose_logger.warning(
f"Failed to retrieve file object for output_file_id={original_output_file_id}: {str(e)}. Storing with None and will fetch on-demand."
)
await self.store_unified_file_id(
file_id=response.output_file_id,
file_object=file_object,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_mappings={model_id: original_output_file_id},
user_api_key_dict=user_api_key_dict,
)
asyncio.create_task(
self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="batch",
user_api_key_dict=user_api_key_dict,
)
await self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="batch",
user_api_key_dict=user_api_key_dict,
)
elif isinstance(response, LiteLLMFineTuningJob):
## Check if unified_file_id is in the response
@ -958,15 +958,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
response.id = self.get_unified_generic_response_id(
model_id=model_id, generic_response_id=response.id
)
asyncio.create_task(
self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="fine-tune",
user_api_key_dict=user_api_key_dict,
)
await self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="fine-tune",
user_api_key_dict=user_api_key_dict,
)
elif isinstance(response, AsyncCursorPage):
"""

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.29"
version = "0.1.31"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.29"
version = "0.1.31"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",

4
litellm-js/spend-logs/package.json
View file

@ -11,6 +11,8 @@
"tsx": "^4.7.1"
},
"overrides": {
"glob": ">=11.1.0"
"glob": ">=11.1.0",
"tar": ">=7.5.7",
"@isaacs/brace-expansion": ">=5.0.1"
}
}

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.32.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.33-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.33.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34.tar.gz vendored Normal file
View file

Binary file not shown.

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260207093506_add_available_on_public_internet_to_mcp_servers/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "available_on_public_internet" BOOLEAN NOT NULL DEFAULT false;

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260207110613_add_soft_budget_to_deleted_teams_table/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN "soft_budget" DOUBLE PRECISION;

8
litellm-proxy-extras/litellm_proxy_extras/migrations/20260209085821_add_verificationtoken_indexes/migration.sql Normal file
View file

@ -0,0 +1,8 @@
-- CreateIndex
CREATE INDEX "LiteLLM_VerificationToken_user_id_team_id_idx" ON "LiteLLM_VerificationToken"("user_id", "team_id");
-- CreateIndex
CREATE INDEX "LiteLLM_VerificationToken_team_id_idx" ON "LiteLLM_VerificationToken"("team_id");
-- CreateIndex
CREATE INDEX "LiteLLM_VerificationToken_budget_reset_at_expires_idx" ON "LiteLLM_VerificationToken"("budget_reset_at", "expires");

13
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -148,6 +148,7 @@ model LiteLLM_DeletedTeamTable {
members_with_roles Json @default("{}")
metadata Json @default("{}")
max_budget Float?
soft_budget Float?
spend Float @default(0.0)
models String[]
max_parallel_requests Int?
@ -263,6 +264,7 @@ model LiteLLM_MCPServerTable {
token_url String?
registration_url String?
allow_all_keys Boolean @default(false)
available_on_public_internet Boolean @default(false)
}
// Generate Tokens for Proxy
@ -308,6 +310,16 @@ model LiteLLM_VerificationToken {
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
// SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub"
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."user_id" = $1 OFFSET $2
@@index([user_id, team_id])
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."team_id" = $1 OFFSET $2
@@index([team_id])
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE (("public"."LiteLLM_VerificationToken"."expires" IS NULL OR "public"."LiteLLM_VerificationToken"."expires" > $1) AND "public"."LiteLLM_VerificationToken"."budget_reset_at" < $2) OFFSET $3
@@index([budget_reset_at, expires])
}
// Audit table for deleted keys - preserves spend and key information for historical tracking
@ -902,6 +914,7 @@ model LiteLLM_PolicyAttachmentTable {
teams String[] @default([]) // Team aliases or patterns
keys String[] @default([]) // Key aliases or patterns
models String[] @default([]) // Model names or patterns
tags String[] @default([]) // Tag patterns (e.g., ["healthcare", "prod-*"])
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.31"
version = "0.4.34"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.31"
version = "0.4.34"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

2
litellm/__init__.py
View file

@ -1155,6 +1155,7 @@ from .exceptions import (
BadRequestError,
ImageFetchError,
NotFoundError,
PermissionDeniedError,
RateLimitError,
ServiceUnavailableError,
BadGatewayError,
@ -1393,6 +1394,7 @@ if TYPE_CHECKING:
from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig
from .llms.volcengine.responses.transformation import VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig
from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig
from .llms.perplexity.responses.transformation import PerplexityResponsesConfig as PerplexityResponsesConfig
from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig
from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config
from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig

6
litellm/_lazy_imports_registry.py
View file

@ -226,6 +226,7 @@ LLM_CONFIG_NAMES = (
"XAIResponsesAPIConfig",
"LiteLLMProxyResponsesAPIConfig",
"VolcEngineResponsesAPIConfig",
"PerplexityResponsesConfig",
"GoogleAIStudioInteractionsConfig",
"OpenAIOSeriesConfig",
"AnthropicSkillsConfig",
@ -274,6 +275,7 @@ LLM_CONFIG_NAMES = (
"LmStudioEmbeddingConfig",
"NscaleConfig",
"PerplexityChatConfig",
"PerplexityResponsesConfig",
"AzureOpenAIO1Config",
"IBMWatsonXAIConfig",
"IBMWatsonXChatConfig",
@ -901,6 +903,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.manus.responses.transformation",
"ManusResponsesAPIConfig",
),
"PerplexityResponsesConfig": (
".llms.perplexity.responses.transformation",
"PerplexityResponsesConfig",
),
"GoogleAIStudioInteractionsConfig": (
".llms.gemini.interactions.transformation",
"GoogleAIStudioInteractionsConfig",

96
litellm/_logging.py
View file

@ -1,9 +1,13 @@
import json
import ast
import logging
import os
import sys
from datetime import datetime
from logging import Formatter
from typing import Any, Dict, Optional
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
set_verbose = False
@ -19,6 +23,67 @@ handler = logging.StreamHandler()
handler.setLevel(numeric_level)
def _try_parse_json_message(message: str) -> Optional[Dict[str, Any]]:
"""
Try to parse a log message as JSON. Returns parsed dict if valid, else None.
Handles messages that are entirely valid JSON (e.g. json.dumps output).
Uses shared safe_json_loads for consistent error handling.
"""
if not message or not isinstance(message, str):
return None
msg_stripped = message.strip()
if not (msg_stripped.startswith("{") or msg_stripped.startswith("[")):
return None
parsed = safe_json_loads(message, default=None)
if parsed is None or not isinstance(parsed, dict):
return None
return parsed
def _try_parse_embedded_python_dict(message: str) -> Optional[Dict[str, Any]]:
"""
Try to find and parse a Python dict repr (e.g. str(d) or repr(d)) embedded in
the message. Handles patterns like:
"get_available_deployment for model: X, Selected deployment: {'model_name': '...', ...} for model: X"
Uses ast.literal_eval for safe parsing. Returns the parsed dict or None.
"""
if not message or not isinstance(message, str) or "{" not in message:
return None
i = 0
while i < len(message):
start = message.find("{", i)
if start == -1:
break
depth = 0
for j in range(start, len(message)):
c = message[j]
if c == "{":
depth += 1
elif c == "}":
depth -= 1
if depth == 0:
substr = message[start : j + 1]
try:
result = ast.literal_eval(substr)
if isinstance(result, dict) and len(result) > 0:
return result
except (ValueError, SyntaxError, TypeError):
pass
break
i = start + 1
return None
# Standard LogRecord attribute names - used to identify 'extra' fields.
# Derived at runtime so we automatically include version-specific attrs (e.g. taskName).
def _get_standard_record_attrs() -> frozenset:
"""Standard LogRecord attribute names - excludes extra keys from logger.debug(..., extra={...})."""
return frozenset(logging.LogRecord("", 0, "", 0, "", (), None).__dict__.keys())
_STANDARD_RECORD_ATTRS = _get_standard_record_attrs()
class JsonFormatter(Formatter):
def __init__(self):
super(JsonFormatter, self).__init__()
@ -29,16 +94,31 @@ class JsonFormatter(Formatter):
return dt.isoformat()
def format(self, record):
json_record = {
"message": record.getMessage(),
message_str = record.getMessage()
json_record: Dict[str, Any] = {
"message": message_str,
"level": record.levelname,
"timestamp": self.formatTime(record),
}
# Parse embedded JSON or Python dict repr in message so sub-fields become first-class properties
parsed = _try_parse_json_message(message_str)
if parsed is None:
parsed = _try_parse_embedded_python_dict(message_str)
if parsed is not None:
for key, value in parsed.items():
if key not in json_record:
json_record[key] = value
# Include extra attributes passed via logger.debug("msg", extra={...})
for key, value in record.__dict__.items():
if key not in _STANDARD_RECORD_ATTRS and key not in json_record:
json_record[key] = value
if record.exc_info:
json_record["stacktrace"] = self.formatException(record.exc_info)
return json.dumps(json_record)
return safe_dumps(json_record)
# Function to set up exception handlers for JSON logging
@ -169,15 +249,15 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
def _get_uvicorn_json_log_config():
"""
Generate a uvicorn log_config dictionary that applies JSON formatting to all loggers.
This ensures that uvicorn's access logs, error logs, and all application logs
are formatted as JSON when json_logs is enabled.
"""
json_formatter_class = "litellm._logging.JsonFormatter"
# Use the module-level log_level variable for consistency
uvicorn_log_level = log_level.upper()
log_config = {
"version": 1,
"disable_existing_loggers": False,
@ -222,7 +302,7 @@ def _get_uvicorn_json_log_config():
},
},
}
return log_config

14
litellm/a2a_protocol/__init__.py
View file

@ -39,6 +39,12 @@ Example usage (class-based):
"""
from litellm.a2a_protocol.client import A2AClient
from litellm.a2a_protocol.exceptions import (
A2AAgentCardError,
A2AConnectionError,
A2AError,
A2ALocalhostURLError,
)
from litellm.a2a_protocol.main import (
aget_agent_card,
asend_message,
@ -49,11 +55,19 @@ from litellm.a2a_protocol.main import (
from litellm.types.agents import LiteLLMSendMessageResponse
__all__ = [
# Client
"A2AClient",
# Functions
"asend_message",
"send_message",
"asend_message_streaming",
"aget_agent_card",
"create_a2a_client",
# Response types
"LiteLLMSendMessageResponse",
# Exceptions
"A2AError",
"A2AConnectionError",
"A2AAgentCardError",
"A2ALocalhostURLError",
]

67
litellm/a2a_protocol/card_resolver.py
View file

@ -7,6 +7,7 @@ Extends the A2A SDK's card resolver to support multiple well-known paths.
from typing import TYPE_CHECKING, Any, Dict, Optional
from litellm._logging import verbose_logger
from litellm.constants import LOCALHOST_URL_PATTERNS
if TYPE_CHECKING:
from a2a.types import AgentCard
@ -26,15 +27,61 @@ except ImportError:
pass
def is_localhost_or_internal_url(url: Optional[str]) -> bool:
"""
Check if a URL is a localhost or internal URL.
This detects common development URLs that are accidentally left in
agent cards when deploying to production.
Args:
url: The URL to check
Returns:
True if the URL is localhost/internal
"""
if not url:
return False
url_lower = url.lower()
return any(pattern in url_lower for pattern in LOCALHOST_URL_PATTERNS)
def fix_agent_card_url(agent_card: "AgentCard", base_url: str) -> "AgentCard":
"""
Fix the agent card URL if it contains a localhost/internal address.
Many A2A agents are deployed with agent cards that contain internal URLs
like "http://0.0.0.0:8001/" or "http://localhost:8000/". This function
replaces such URLs with the provided base_url.
Args:
agent_card: The agent card to fix
base_url: The base URL to use as replacement
Returns:
The agent card with the URL fixed if necessary
"""
card_url = getattr(agent_card, "url", None)
if card_url and is_localhost_or_internal_url(card_url):
# Normalize base_url to ensure it ends with /
fixed_url = base_url.rstrip("/") + "/"
agent_card.url = fixed_url
return agent_card
class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
"""
Custom A2A card resolver that supports multiple well-known paths.
Extends the base A2ACardResolver to try both:
- /.well-known/agent-card.json (standard)
- /.well-known/agent.json (previous/alternative)
"""
async def get_agent_card(
self,
relative_card_path: Optional[str] = None,
@ -42,17 +89,17 @@ class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
) -> "AgentCard":
"""
Fetch the agent card, trying multiple well-known paths.
First tries the standard path, then falls back to the previous path.
Args:
relative_card_path: Optional path to the agent card endpoint.
If None, tries both well-known paths.
http_kwargs: Optional dictionary of keyword arguments to pass to httpx.get
Returns:
AgentCard from the A2A agent
Raises:
A2AClientHTTPError or A2AClientJSONError if both paths fail
"""
@ -62,13 +109,13 @@ class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
relative_card_path=relative_card_path,
http_kwargs=http_kwargs,
)
# Try both well-known paths
paths = [
AGENT_CARD_WELL_KNOWN_PATH,
PREV_AGENT_CARD_WELL_KNOWN_PATH,
]
last_error = None
for path in paths:
try:
@ -85,11 +132,11 @@ class LiteLLMA2ACardResolver(_A2ACardResolver): # type: ignore[misc]
)
last_error = e
continue
# If we get here, all paths failed - re-raise the last error
if last_error is not None:
raise last_error
# This shouldn't happen, but just in case
raise Exception(
f"Failed to fetch agent card from {self.base_url}. "

203
litellm/a2a_protocol/exception_mapping_utils.py Normal file
View file

@ -0,0 +1,203 @@
"""
A2A Protocol Exception Mapping Utils.
Maps A2A SDK exceptions to LiteLLM A2A exception types.
"""
from typing import TYPE_CHECKING, Any, Optional
from litellm._logging import verbose_logger
from litellm.a2a_protocol.card_resolver import (
fix_agent_card_url,
is_localhost_or_internal_url,
)
from litellm.a2a_protocol.exceptions import (
A2AAgentCardError,
A2AConnectionError,
A2AError,
A2ALocalhostURLError,
)
from litellm.constants import CONNECTION_ERROR_PATTERNS
if TYPE_CHECKING:
from a2a.client import A2AClient as A2AClientType
# Runtime import
A2A_SDK_AVAILABLE = False
try:
from a2a.client import A2AClient as _A2AClient # type: ignore[no-redef]
A2A_SDK_AVAILABLE = True
except ImportError:
_A2AClient = None # type: ignore[assignment, misc]
class A2AExceptionCheckers:
"""
Helper class for checking various A2A error conditions.
"""
@staticmethod
def is_connection_error(error_str: str) -> bool:
"""
Check if an error string indicates a connection error.
Args:
error_str: The error string to check
Returns:
True if the error indicates a connection issue
"""
if not isinstance(error_str, str):
return False
error_str_lower = error_str.lower()
return any(pattern in error_str_lower for pattern in CONNECTION_ERROR_PATTERNS)
@staticmethod
def is_localhost_url(url: Optional[str]) -> bool:
"""
Check if a URL is a localhost/internal URL.
Args:
url: The URL to check
Returns:
True if the URL is localhost/internal
"""
return is_localhost_or_internal_url(url)
@staticmethod
def is_agent_card_error(error_str: str) -> bool:
"""
Check if an error string indicates an agent card error.
Args:
error_str: The error string to check
Returns:
True if the error is related to agent card fetching/parsing
"""
if not isinstance(error_str, str):
return False
error_str_lower = error_str.lower()
agent_card_patterns = [
"agent card",
"agent-card",
".well-known",
"card not found",
"invalid agent",
]
return any(pattern in error_str_lower for pattern in agent_card_patterns)
def map_a2a_exception(
original_exception: Exception,
card_url: Optional[str] = None,
api_base: Optional[str] = None,
model: Optional[str] = None,
) -> Exception:
"""
Map an A2A SDK exception to a LiteLLM A2A exception type.
Args:
original_exception: The original exception from the A2A SDK
card_url: The URL from the agent card (if available)
api_base: The original API base URL
model: The model/agent name
Returns:
A mapped LiteLLM A2A exception
Raises:
A2ALocalhostURLError: If the error is a connection error to a localhost URL
A2AConnectionError: If the error is a general connection error
A2AAgentCardError: If the error is related to agent card issues
A2AError: For other A2A-related errors
"""
error_str = str(original_exception)
# Check for localhost URL connection error (special case - retryable)
if (
card_url
and api_base
and A2AExceptionCheckers.is_localhost_url(card_url)
and A2AExceptionCheckers.is_connection_error(error_str)
):
raise A2ALocalhostURLError(
localhost_url=card_url,
base_url=api_base,
original_error=original_exception,
model=model,
)
# Check for agent card errors
if A2AExceptionCheckers.is_agent_card_error(error_str):
raise A2AAgentCardError(
message=error_str,
url=api_base,
model=model,
)
# Check for general connection errors
if A2AExceptionCheckers.is_connection_error(error_str):
raise A2AConnectionError(
message=error_str,
url=card_url or api_base,
model=model,
)
# Default: wrap in generic A2AError
raise A2AError(
message=error_str,
model=model,
)
def handle_a2a_localhost_retry(
error: A2ALocalhostURLError,
agent_card: Any,
a2a_client: "A2AClientType",
is_streaming: bool = False,
) -> "A2AClientType":
"""
Handle A2ALocalhostURLError by fixing the URL and creating a new client.
This is called when we catch an A2ALocalhostURLError and want to retry
with the corrected URL.
Args:
error: The localhost URL error
agent_card: The agent card object to fix
a2a_client: The current A2A client
is_streaming: Whether this is a streaming request (for logging)
Returns:
A new A2A client with the fixed URL
Raises:
ImportError: If the A2A SDK is not installed
"""
if not A2A_SDK_AVAILABLE or _A2AClient is None:
raise ImportError(
"A2A SDK is required for localhost retry handling. "
"Install it with: pip install a2a"
)
request_type = "streaming " if is_streaming else ""
verbose_logger.warning(
f"A2A {request_type}request to '{error.localhost_url}' failed: {error.original_error}. "
f"Agent card contains localhost/internal URL. "
f"Retrying with base_url '{error.base_url}'."
)
# Fix the agent card URL
fix_agent_card_url(agent_card, error.base_url)
# Create a new client with the fixed agent card (transport caches URL)
return _A2AClient(
httpx_client=a2a_client._transport.httpx_client, # type: ignore[union-attr]
agent_card=agent_card,
)

150
litellm/a2a_protocol/exceptions.py Normal file
View file

@ -0,0 +1,150 @@
"""
A2A Protocol Exceptions.
Custom exception types for A2A protocol operations, following LiteLLM's exception pattern.
"""
from typing import Optional
import httpx
class A2AError(Exception):
"""
Base exception for A2A protocol errors.
Follows the same pattern as LiteLLM's main exceptions.
"""
def __init__(
self,
message: str,
status_code: int = 500,
llm_provider: str = "a2a_agent",
model: Optional[str] = None,
response: Optional[httpx.Response] = None,
litellm_debug_info: Optional[str] = None,
max_retries: Optional[int] = None,
num_retries: Optional[int] = None,
):
self.status_code = status_code
self.message = f"litellm.A2AError: {message}"
self.llm_provider = llm_provider
self.model = model
self.litellm_debug_info = litellm_debug_info
self.max_retries = max_retries
self.num_retries = num_retries
self.response = response or httpx.Response(
status_code=self.status_code,
request=httpx.Request(method="POST", url="https://litellm.ai"),
)
super().__init__(self.message)
def __str__(self) -> str:
_message = self.message
if self.num_retries:
_message += f" LiteLLM Retried: {self.num_retries} times"
if self.max_retries:
_message += f", LiteLLM Max Retries: {self.max_retries}"
return _message
def __repr__(self) -> str:
return self.__str__()
class A2AConnectionError(A2AError):
"""
Raised when connection to an A2A agent fails.
This typically occurs when:
- The agent is unreachable
- The agent card contains a localhost/internal URL
- Network issues prevent connection
"""
def __init__(
self,
message: str,
url: Optional[str] = None,
model: Optional[str] = None,
response: Optional[httpx.Response] = None,
litellm_debug_info: Optional[str] = None,
max_retries: Optional[int] = None,
num_retries: Optional[int] = None,
):
self.url = url
super().__init__(
message=message,
status_code=503,
llm_provider="a2a_agent",
model=model,
response=response,
litellm_debug_info=litellm_debug_info,
max_retries=max_retries,
num_retries=num_retries,
)
class A2AAgentCardError(A2AError):
"""
Raised when there's an issue with the agent card.
This includes:
- Failed to fetch agent card
- Invalid agent card format
- Missing required fields
"""
def __init__(
self,
message: str,
url: Optional[str] = None,
model: Optional[str] = None,
response: Optional[httpx.Response] = None,
litellm_debug_info: Optional[str] = None,
):
self.url = url
super().__init__(
message=message,
status_code=404,
llm_provider="a2a_agent",
model=model,
response=response,
litellm_debug_info=litellm_debug_info,
)
class A2ALocalhostURLError(A2AConnectionError):
"""
Raised when an agent card contains a localhost/internal URL.
Many A2A agents are deployed with agent cards that contain internal URLs
like "http://0.0.0.0:8001/" or "http://localhost:8000/". This error
indicates that the URL needs to be corrected and the request should be retried.
Attributes:
localhost_url: The localhost/internal URL found in the agent card
base_url: The public base URL that should be used instead
original_error: The original connection error that was raised
"""
def __init__(
self,
localhost_url: str,
base_url: str,
original_error: Optional[Exception] = None,
model: Optional[str] = None,
):
self.localhost_url = localhost_url
self.base_url = base_url
self.original_error = original_error
message = (
f"Agent card contains localhost/internal URL '{localhost_url}'. "
f"Retrying with base URL '{base_url}'."
)
super().__init__(
message=message,
url=localhost_url,
model=model,
)

184
litellm/a2a_protocol/main.py
View file

@ -44,6 +44,11 @@ except ImportError:
# Import our custom card resolver that supports multiple well-known paths
from litellm.a2a_protocol.card_resolver import LiteLLMA2ACardResolver
from litellm.a2a_protocol.exception_mapping_utils import (
handle_a2a_localhost_retry,
map_a2a_exception,
)
from litellm.a2a_protocol.exceptions import A2ALocalhostURLError
# Use our custom resolver instead of the default A2A SDK resolver
A2ACardResolver = LiteLLMA2ACardResolver
@ -244,10 +249,50 @@ async def asend_message(
verbose_logger.info(f"A2A send_message request_id={request.id}, agent={agent_name}")
a2a_response = await a2a_client.send_message(request)
# Get agent card URL for localhost retry logic
agent_card = getattr(a2a_client, "_litellm_agent_card", None) or getattr(
a2a_client, "agent_card", None
)
card_url = getattr(agent_card, "url", None) if agent_card else None
# Retry loop: if connection fails due to localhost URL in agent card, retry with fixed URL
a2a_response = None
for _ in range(2): # max 2 attempts: original + 1 retry
try:
a2a_response = await a2a_client.send_message(request)
break # success, exit retry loop
except A2ALocalhostURLError as e:
# Localhost URL error - fix and retry
a2a_client = handle_a2a_localhost_retry(
error=e,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=False,
)
card_url = agent_card.url if agent_card else None
except Exception as e:
# Map exception - will raise A2ALocalhostURLError if applicable
try:
map_a2a_exception(e, card_url, api_base, model=agent_name)
except A2ALocalhostURLError as localhost_err:
# Localhost URL error - fix and retry
a2a_client = handle_a2a_localhost_retry(
error=localhost_err,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=False,
)
card_url = agent_card.url if agent_card else None
continue
except Exception:
# Re-raise the mapped exception
raise
verbose_logger.info(f"A2A send_message completed, request_id={request.id}")
# a2a_response is guaranteed to be set if we reach here (loop breaks on success or raises)
assert a2a_response is not None
# Wrap in LiteLLM response type for _hidden_params support
response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response)
@ -307,6 +352,48 @@ def send_message(
)
def _build_streaming_logging_obj(
request: "SendStreamingMessageRequest",
agent_name: str,
agent_id: Optional[str],
litellm_params: Optional[Dict[str, Any]],
metadata: Optional[Dict[str, Any]],
proxy_server_request: Optional[Dict[str, Any]],
) -> Logging:
"""Build logging object for streaming A2A requests."""
start_time = datetime.datetime.now()
model = f"a2a_agent/{agent_name}"
logging_obj = Logging(
model=model,
messages=[{"role": "user", "content": "streaming-request"}],
stream=False,
call_type="asend_message_streaming",
start_time=start_time,
litellm_call_id=str(request.id),
function_id=str(request.id),
)
logging_obj.model = model
logging_obj.custom_llm_provider = "a2a_agent"
logging_obj.model_call_details["model"] = model
logging_obj.model_call_details["custom_llm_provider"] = "a2a_agent"
if agent_id:
logging_obj.model_call_details["agent_id"] = agent_id
_litellm_params = litellm_params.copy() if litellm_params else {}
if metadata:
_litellm_params["metadata"] = metadata
if proxy_server_request:
_litellm_params["proxy_server_request"] = proxy_server_request
logging_obj.litellm_params = _litellm_params
logging_obj.optional_params = _litellm_params
logging_obj.model_call_details["litellm_params"] = _litellm_params
logging_obj.model_call_details["metadata"] = metadata or {}
return logging_obj
async def asend_message_streaming(
a2a_client: Optional["A2AClientType"] = None,
request: Optional["SendStreamingMessageRequest"] = None,
@ -403,55 +490,72 @@ async def asend_message_streaming(
verbose_logger.info(f"A2A send_message_streaming request_id={request.id}")
# Track for logging
start_time = datetime.datetime.now()
stream = a2a_client.send_message_streaming(request)
# Build logging object for streaming completion callbacks
agent_card = getattr(a2a_client, "_litellm_agent_card", None) or getattr(
a2a_client, "agent_card", None
)
card_url = getattr(agent_card, "url", None) if agent_card else None
agent_name = getattr(agent_card, "name", "unknown") if agent_card else "unknown"
model = f"a2a_agent/{agent_name}"
logging_obj = Logging(
model=model,
messages=[{"role": "user", "content": "streaming-request"}],
stream=False, # complete response logging after stream ends
call_type="asend_message_streaming",
start_time=start_time,
litellm_call_id=str(request.id),
function_id=str(request.id),
)
logging_obj.model = model
logging_obj.custom_llm_provider = "a2a_agent"
logging_obj.model_call_details["model"] = model
logging_obj.model_call_details["custom_llm_provider"] = "a2a_agent"
if agent_id:
logging_obj.model_call_details["agent_id"] = agent_id
# Propagate litellm_params for spend logging (includes cost_per_query, etc.)
_litellm_params = litellm_params.copy() if litellm_params else {}
# Merge metadata into litellm_params.metadata (required for proxy cost tracking)
if metadata:
_litellm_params["metadata"] = metadata
if proxy_server_request:
_litellm_params["proxy_server_request"] = proxy_server_request
logging_obj.litellm_params = _litellm_params
logging_obj.optional_params = _litellm_params # used by cost calc
logging_obj.model_call_details["litellm_params"] = _litellm_params
logging_obj.model_call_details["metadata"] = metadata or {}
iterator = A2AStreamingIterator(
stream=stream,
logging_obj = _build_streaming_logging_obj(
request=request,
logging_obj=logging_obj,
agent_name=agent_name,
agent_id=agent_id,
litellm_params=litellm_params,
metadata=metadata,
proxy_server_request=proxy_server_request,
)
async for chunk in iterator:
yield chunk
# Retry loop: if connection fails due to localhost URL in agent card, retry with fixed URL
# Connection errors in streaming typically occur on first chunk iteration
first_chunk = True
for attempt in range(2): # max 2 attempts: original + 1 retry
stream = a2a_client.send_message_streaming(request)
iterator = A2AStreamingIterator(
stream=stream,
request=request,
logging_obj=logging_obj,
agent_name=agent_name,
)
try:
first_chunk = True
async for chunk in iterator:
if first_chunk:
first_chunk = False # connection succeeded
yield chunk
return # stream completed successfully
except A2ALocalhostURLError as e:
# Only retry on first chunk, not mid-stream
if first_chunk and attempt == 0:
a2a_client = handle_a2a_localhost_retry(
error=e,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=True,
)
card_url = agent_card.url if agent_card else None
else:
raise
except Exception as e:
# Only map exception on first chunk
if first_chunk and attempt == 0:
try:
map_a2a_exception(e, card_url, api_base, model=agent_name)
except A2ALocalhostURLError as localhost_err:
# Localhost URL error - fix and retry
a2a_client = handle_a2a_localhost_retry(
error=localhost_err,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=True,
)
card_url = agent_card.url if agent_card else None
continue
except Exception:
# Re-raise the mapped exception
raise
raise
async def create_a2a_client(

151
litellm/anthropic_beta_headers_config.json Normal file
View file

@ -0,0 +1,151 @@
{
"description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.",
"anthropic": {
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"bash_20241022": "bash_20241022",
"bash_20250124": "bash_20250124",
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": "compact-2026-01-12",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": "fast-mode-2026-02-01",
"files-api-2025-04-14": "files-api-2025-04-14",
"structured-output-2024-03-01": "structured-output-2024-03-01",
"fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-servers-2025-12-04": "mcp-servers-2025-12-04",
"output-128k-2025-02-19": "output-128k-2025-02-19",
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": "text_editor_20241022",
"text_editor_20250124": "text_editor_20250124",
"token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
"web-search-2025-03-05": "web-search-2025-03-05"
},
"azure_ai": {
"advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
"bash_20241022": "bash_20241022",
"bash_20250124": "bash_20250124",
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": null,
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": "files-api-2025-04-14",
"fine-grained-tool-streaming-2025-05-14": null,
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": "mcp-client-2025-11-20",
"mcp-client-2025-04-04": "mcp-client-2025-04-04",
"mcp-servers-2025-12-04": "mcp-servers-2025-12-04",
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"token-efficient-tools-2025-02-19": null,
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
"web-search-2025-03-05": "web-search-2025-03-05"
},
"bedrock_converse": {
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": null,
"compact-2026-01-12": null,
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": null,
"context-management-2025-06-27": "context-management-2025-06-27",
"effort-2025-11-24": null,
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": null,
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"token-efficient-tools-2025-02-19": null,
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": null
},
"bedrock": {
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": null,
"compact-2026-01-12": "compact-2026-01-12",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"effort-2025-11-24": null,
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": null,
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": null,
"text_editor_20241022": null,
"text_editor_20250124": null,
"token-efficient-tools-2025-02-19": null,
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": null
},
"vertex_ai": {
"advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",
"bash_20241022": null,
"bash_20250124": null,
"code-execution-2025-08-25": null,
"compact-2026-01-12": null,
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": null,
"context-management-2025-06-27": "context-management-2025-06-27",
"effort-2025-11-24": null,
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": null,
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": null,
"text_editor_20241022": null,
"text_editor_20250124": null,
"token-efficient-tools-2025-02-19": null,
"tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
"web-fetch-2025-09-10": null,
"web-search-2025-03-05": "web-search-2025-03-05"
}
}

237
litellm/anthropic_beta_headers_manager.py Normal file
View file

@ -0,0 +1,237 @@
"""
Centralized manager for Anthropic beta headers across different providers.
This module provides utilities to:
1. Load beta header configuration from JSON (mapping of supported headers per provider)
2. Filter and map beta headers based on provider support
3. Handle provider-specific header name mappings (e.g., advanced-tool-use -> tool-search-tool)
Design:
- JSON config contains mapping of beta headers for each provider
- Keys are input header names, values are provider-specific header names (or null if unsupported)
- Only headers present in mapping keys with non-null values can be forwarded
- This enforces stricter validation than the previous unsupported list approach
"""
import json
import os
from typing import Dict, List, Optional, Set
from litellm.litellm_core_utils.litellm_logging import verbose_logger
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: Optional[Dict] = None
def _load_beta_headers_config() -> Dict:
"""
Load the beta headers configuration from JSON file.
Uses caching to avoid repeated file reads.
Returns:
Dict containing the beta headers configuration
"""
global _BETA_HEADERS_CONFIG
if _BETA_HEADERS_CONFIG is not None:
return _BETA_HEADERS_CONFIG
config_path = os.path.join(
os.path.dirname(__file__),
"anthropic_beta_headers_config.json"
)
try:
with open(config_path, "r") as f:
_BETA_HEADERS_CONFIG = json.load(f)
verbose_logger.debug(f"Loaded beta headers config from {config_path}")
return _BETA_HEADERS_CONFIG
except Exception as e:
verbose_logger.error(f"Failed to load beta headers config: {e}")
# Return empty config as fallback (empty mappings)
return {
"anthropic": {},
"azure_ai": {},
"bedrock": {},
"bedrock_converse": {},
"vertex_ai": {}
}
def get_provider_name(provider: str) -> str:
"""
Resolve provider aliases to canonical provider names.
Args:
provider: Provider name (may be an alias)
Returns:
Canonical provider name
"""
config = _load_beta_headers_config()
aliases = config.get("provider_aliases", {})
return aliases.get(provider, provider)
def filter_and_transform_beta_headers(
beta_headers: List[str],
provider: str,
) -> List[str]:
"""
Filter and transform beta headers based on provider's mapping configuration.
This function:
1. Only allows headers that are present in the provider's mapping keys
2. Filters out headers with null values (unsupported)
3. Maps headers to provider-specific names (e.g., advanced-tool-use -> tool-search-tool)
Args:
beta_headers: List of Anthropic beta header values
provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai")
Returns:
List of filtered and transformed beta headers for the provider
"""
if not beta_headers:
return []
config = _load_beta_headers_config()
provider = get_provider_name(provider)
# Get the header mapping for this provider
provider_mapping = config.get(provider, {})
filtered_headers: Set[str] = set()
for header in beta_headers:
header = header.strip()
# Check if header is in the mapping
if header not in provider_mapping:
verbose_logger.debug(
f"Dropping unknown beta header '{header}' for provider '{provider}' (not in mapping)"
)
continue
# Get the mapped header value
mapped_header = provider_mapping[header]
# Skip if header is unsupported (null value)
if mapped_header is None:
verbose_logger.debug(
f"Dropping unsupported beta header '{header}' for provider '{provider}'"
)
continue
# Add the mapped header
filtered_headers.add(mapped_header)
return sorted(list(filtered_headers))
def is_beta_header_supported(
beta_header: str,
provider: str,
) -> bool:
"""
Check if a specific beta header is supported by a provider.
Args:
beta_header: The Anthropic beta header value
provider: Provider name
Returns:
True if the header is in the mapping with a non-null value, False otherwise
"""
config = _load_beta_headers_config()
provider = get_provider_name(provider)
provider_mapping = config.get(provider, {})
# Header is supported if it's in the mapping and has a non-null value
return beta_header in provider_mapping and provider_mapping[beta_header] is not None
def get_provider_beta_header(
anthropic_beta_header: str,
provider: str,
) -> Optional[str]:
"""
Get the provider-specific beta header name for a given Anthropic beta header.
This function handles header transformations/mappings (e.g., advanced-tool-use -> tool-search-tool).
Args:
anthropic_beta_header: The Anthropic beta header value
provider: Provider name
Returns:
The provider-specific header name if supported, or None if unsupported/unknown
"""
config = _load_beta_headers_config()
provider = get_provider_name(provider)
# Get the header mapping for this provider
provider_mapping = config.get(provider, {})
# Check if header is in the mapping
if anthropic_beta_header not in provider_mapping:
return None
# Return the mapped value (could be None if unsupported)
return provider_mapping[anthropic_beta_header]
def update_headers_with_filtered_beta(
headers: dict,
provider: str,
) -> dict:
"""
Update headers dict by filtering and transforming anthropic-beta header values.
Modifies the headers dict in place and returns it.
Args:
headers: Request headers dict (will be modified in place)
provider: Provider name
Returns:
Updated headers dict
"""
existing_beta = headers.get("anthropic-beta")
if not existing_beta:
return headers
# Parse existing beta headers
beta_values = [b.strip() for b in existing_beta.split(",") if b.strip()]
# Filter and transform based on provider
filtered_beta_values = filter_and_transform_beta_headers(
beta_headers=beta_values,
provider=provider,
)
# Update or remove the header
if filtered_beta_values:
headers["anthropic-beta"] = ",".join(filtered_beta_values)
else:
# Remove the header if no values remain
headers.pop("anthropic-beta", None)
return headers
def get_unsupported_headers(provider: str) -> List[str]:
"""
Get all beta headers that are unsupported by a provider (have null values in mapping).
Args:
provider: Provider name
Returns:
List of unsupported Anthropic beta header names
"""
config = _load_beta_headers_config()
provider = get_provider_name(provider)
provider_mapping = config.get(provider, {})
# Return headers with null values
return [header for header, value in provider_mapping.items() if value is None]

32
litellm/batch_completion/main.py
View file

@ -237,17 +237,37 @@ def batch_completion_models_all_responses(*args, **kwargs):
if "model" in kwargs:
kwargs.pop("model")
if "models" in kwargs:
models = kwargs["models"]
kwargs.pop("models")
models = kwargs.pop("models")
else:
raise Exception("'models' param not in kwargs")
if isinstance(models, str):
models = [models]
elif isinstance(models, (list, tuple)):
models = list(models)
else:
raise TypeError("'models' must be a string or list of strings")
if len(models) == 0:
return []
responses = []
with concurrent.futures.ThreadPoolExecutor(max_workers=len(models)) as executor:
for idx, model in enumerate(models):
future = executor.submit(litellm.completion, *args, model=model, **kwargs)
if future.result() is not None:
responses.append(future.result())
futures = [
executor.submit(litellm.completion, *args, model=model, **kwargs)
for model in models
]
for future in futures:
try:
result = future.result()
if result is not None:
responses.append(result)
except Exception as e:
print_verbose(
f"batch_completion_models_all_responses: model request failed: {str(e)}"
)
continue
return responses

2
litellm/caching/redis_cache.py
View file

@ -1123,7 +1123,7 @@ class RedisCache(BaseCache):
redis_client = redis_async.Redis(**self.redis_kwargs)
# Test the connection
ping_result = await redis_client.ping()
ping_result = await redis_client.ping() # type: ignore[misc]
# Close the connection
await redis_client.aclose() # type: ignore[attr-defined]

2
litellm/caching/redis_cluster_cache.py
View file

@ -83,7 +83,7 @@ class RedisClusterCache(RedisCache):
)
# Test the connection
ping_result = await redis_client.ping() # type: ignore[attr-defined]
ping_result = await redis_client.ping() # type: ignore[attr-defined, misc]
# Close the connection
await redis_client.aclose() # type: ignore[attr-defined]

52
litellm/constants.py
View file

@ -2,6 +2,8 @@ import os
import sys
from typing import List, Literal
from litellm.litellm_core_utils.env_utils import get_env_int
DEFAULT_HEALTH_CHECK_PROMPT = str(
os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")
)
@ -46,6 +48,14 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = int(
os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)
)
DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
# Model cost map validation constants
MODEL_COST_MAP_MIN_MODEL_COUNT = int(
os.getenv("MODEL_COST_MAP_MIN_MODEL_COUNT", 50)
) # Minimum number of models a fetched cost map must contain to be considered valid
MODEL_COST_MAP_MAX_SHRINK_RATIO = float(
os.getenv("MODEL_COST_MAP_MAX_SHRINK_RATIO", 0.5)
) # Maximum allowed shrinkage ratio vs local backup (0.5 = reject if fetched map is <50% of backup)
DEFAULT_IMAGE_WIDTH = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300))
DEFAULT_IMAGE_HEIGHT = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300))
# Maximum size for image URL downloads in MB (default 50MB, set to 0 to disable limit)
@ -81,6 +91,20 @@ MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH = int(
os.getenv("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150)
)
# MCP OAuth2 Client Credentials Defaults
MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS = int(
os.getenv("MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS", "60")
)
MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE = int(
os.getenv("MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE", "200")
)
MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL = int(
os.getenv("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", "3600")
)
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL = int(
os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10")
)
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
"x-litellm-semantic-filter-tools",
@ -99,6 +123,11 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int(
)
)
# Maximum number of callbacks that can be registered
# This prevents callbacks from exponentially growing and consuming CPU resources
# Override with LITELLM_MAX_CALLBACKS env var for large deployments (e.g., many teams with guardrails)
MAX_CALLBACKS = get_env_int("LITELLM_MAX_CALLBACKS", 30)
# Generic fallback for unknown models
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int(
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128)
@ -184,6 +213,10 @@ REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_agent_spend_update_bu
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_tag_spend_update_buffer"
MAX_REDIS_BUFFER_DEQUEUE_COUNT = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100))
MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 2000))
# Bounds asyncio.Queue() instances (log queues, spend update queues, etc.) to prevent unbounded memory growth
LITELLM_ASYNCIO_QUEUE_MAXSIZE = int(
os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000)
)
MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int(
os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000)
)
@ -306,6 +339,22 @@ DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2
#### Networking settings ####
request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds
DEFAULT_A2A_AGENT_TIMEOUT: float = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)) # 10 minutes
# Patterns that indicate a localhost/internal URL in A2A agent cards that should be
# replaced with the original base_url. This is a common misconfiguration where
# developers deploy agents with development URLs in their agent cards.
LOCALHOST_URL_PATTERNS: List[str] = [
"localhost",
"127.0.0.1",
"0.0.0.0",
"[::1]", # IPv6 localhost
]
# Patterns in error messages that indicate a connection failure
CONNECTION_ERROR_PATTERNS: List[str] = [
"connect",
"connection",
"network",
"refused",
]
STREAM_SSE_DONE_STRING: str = "[DONE]"
STREAM_SSE_DATA_PREFIX: str = "data: "
### SPEND TRACKING ###
@ -1261,6 +1310,9 @@ DEFAULT_SLACK_ALERTING_THRESHOLD = int(
os.getenv("DEFAULT_SLACK_ALERTING_THRESHOLD", 300)
)
MAX_TEAM_LIST_LIMIT = int(os.getenv("MAX_TEAM_LIST_LIMIT", 20))
MAX_POLICY_ESTIMATE_IMPACT_ROWS = int(
os.getenv("MAX_POLICY_ESTIMATE_IMPACT_ROWS", 1000)
)
DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(
os.getenv("DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD", 0.7)
)

113
litellm/cost_calculator.py
View file

@ -1,5 +1,6 @@
# What is this?
## File for 'response_cost' calculation in Logging
import logging
import time
from functools import lru_cache
from typing import TYPE_CHECKING, Any, List, Literal, Optional, Tuple, Union, cast
@ -774,10 +775,11 @@ def _apply_cost_discount(
discount_amount = original_cost * discount_percent
final_cost = original_cost - discount_amount
verbose_logger.debug(
f"Applied {discount_percent*100}% discount to {custom_llm_provider}: "
f"${original_cost:.6f} -> ${final_cost:.6f} (saved ${discount_amount:.6f})"
)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
f"Applied {discount_percent*100}% discount to {custom_llm_provider}: "
f"${original_cost:.6f} -> ${final_cost:.6f} (saved ${discount_amount:.6f})"
)
return final_cost, discount_percent, discount_amount
@ -807,17 +809,20 @@ def _apply_cost_margin(
margin_config = None
if custom_llm_provider and custom_llm_provider in litellm.cost_margin_config:
margin_config = litellm.cost_margin_config[custom_llm_provider]
verbose_logger.debug(
f"Found provider-specific margin config for {custom_llm_provider}: {margin_config}"
)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
f"Found provider-specific margin config for {custom_llm_provider}: {margin_config}"
)
elif "global" in litellm.cost_margin_config:
margin_config = litellm.cost_margin_config["global"]
verbose_logger.debug(f"Using global margin config: {margin_config}")
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(f"Using global margin config: {margin_config}")
else:
verbose_logger.debug(
f"No margin config found. Provider: {custom_llm_provider}, "
f"Available configs: {list(litellm.cost_margin_config.keys())}"
)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
f"No margin config found. Provider: {custom_llm_provider}, "
f"Available configs: {list(litellm.cost_margin_config.keys())}"
)
if margin_config is not None:
# Handle different margin config formats
@ -836,11 +841,12 @@ def _apply_cost_margin(
final_cost = original_cost + margin_total_amount
verbose_logger.debug(
f"Applied margin to {custom_llm_provider or 'global'}: "
f"${original_cost:.6f} -> ${final_cost:.6f} "
f"(margin: {margin_percent*100 if margin_percent > 0 else 0}% + ${margin_fixed_amount:.6f} = ${margin_total_amount:.6f})"
)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
f"Applied margin to {custom_llm_provider or 'global'}: "
f"${original_cost:.6f} -> ${final_cost:.6f} "
f"(margin: {margin_percent*100 if margin_percent > 0 else 0}% + ${margin_fixed_amount:.6f} = ${margin_total_amount:.6f})"
)
return final_cost, margin_percent, margin_fixed_amount, margin_total_amount
@ -1021,9 +1027,10 @@ def completion_cost( # noqa: PLR0915
for idx, model in enumerate(potential_model_names):
try:
verbose_logger.debug(
f"selected model name for cost calculation: {model}"
)
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug(
f"selected model name for cost calculation: {model}"
)
if completion_response is not None and (
isinstance(completion_response, BaseModel)
@ -1411,37 +1418,47 @@ def completion_cost( # noqa: PLR0915
# Apply discount from module-level config if configured
original_cost = _final_cost
_final_cost, discount_percent, discount_amount = _apply_cost_discount(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
if litellm.cost_discount_config:
_final_cost, discount_percent, discount_amount = _apply_cost_discount(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
else:
discount_percent = 0.0
discount_amount = 0.0
# Apply margin from module-level config if configured
(
_final_cost,
margin_percent,
margin_fixed_amount,
margin_total_amount,
) = _apply_cost_margin(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
if litellm.cost_margin_config:
(
_final_cost,
margin_percent,
margin_fixed_amount,
margin_total_amount,
) = _apply_cost_margin(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
else:
margin_percent = 0.0
margin_fixed_amount = 0.0
margin_total_amount = 0.0
# Store cost breakdown in logging object if available
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar=completion_tokens_cost_usd_dollar,
cost_for_built_in_tools_cost_usd_dollar=cost_for_built_in_tools,
total_cost_usd_dollar=_final_cost,
additional_costs=additional_costs,
original_cost=original_cost,
discount_percent=discount_percent,
discount_amount=discount_amount,
margin_percent=margin_percent,
margin_fixed_amount=margin_fixed_amount,
margin_total_amount=margin_total_amount,
)
if litellm_logging_obj is not None:
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar=completion_tokens_cost_usd_dollar,
cost_for_built_in_tools_cost_usd_dollar=cost_for_built_in_tools,
total_cost_usd_dollar=_final_cost,
original_cost=original_cost,
additional_costs=additional_costs,
discount_percent=discount_percent,
discount_amount=discount_amount,
margin_percent=margin_percent,
margin_fixed_amount=margin_fixed_amount,
margin_total_amount=margin_total_amount,
)
return _final_cost
except Exception as e:
@ -2116,3 +2133,5 @@ def handle_realtime_stream_cost_calculation(
total_cost = input_cost_per_token + output_cost_per_token
return total_cost

2
litellm/experimental_mcp_client/client.py
View file

@ -209,6 +209,8 @@ class MCPClient:
headers["X-API-Key"] = self._mcp_auth_value
elif self.auth_type == MCPAuth.authorization:
headers["Authorization"] = self._mcp_auth_value
elif self.auth_type == MCPAuth.oauth2:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
elif isinstance(self._mcp_auth_value, dict):
headers.update(self._mcp_auth_value)

35
litellm/integrations/arize/arize.py
View file

@ -28,6 +28,41 @@ else:
class ArizeLogger(OpenTelemetry):
"""
Arize logger that sends traces to an Arize endpoint.
Creates its own dedicated TracerProvider so it can coexist with the
generic ``otel`` callback (or any other OTEL-based integration) without
fighting over the global ``opentelemetry.trace`` TracerProvider singleton.
"""
def _init_tracing(self, tracer_provider):
"""
Override to always create a *private* TracerProvider for Arize.
See ArizePhoenixLogger._init_tracing for full rationale.
"""
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import SpanKind
if tracer_provider is not None:
self.tracer = tracer_provider.get_tracer("litellm")
self.span_kind = SpanKind
return
provider = TracerProvider(resource=self._get_litellm_resource(self.config))
provider.add_span_processor(self._get_span_processor())
self.tracer = provider.get_tracer("litellm")
self.span_kind = SpanKind
def _init_otel_logger_on_litellm_proxy(self):
"""
Override: Arize should NOT overwrite the proxy's
``open_telemetry_logger``. That attribute is reserved for the
primary ``otel`` callback which handles proxy-level parent spans.
"""
pass
def set_attributes(self, span: Span, kwargs, response_obj: Optional[Any]):
ArizeLogger.set_arize_attributes(span, kwargs, response_obj)
return

186
litellm/integrations/arize/arize_phoenix.py
View file

@ -5,43 +5,211 @@ from litellm._logging import verbose_logger
from litellm.integrations.arize import _utils
from litellm.integrations.arize._utils import ArizeOTELAttributes
from litellm.types.integrations.arize_phoenix import ArizePhoenixConfig
from litellm.integrations.opentelemetry import OpenTelemetry
if TYPE_CHECKING:
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import Span as _Span
from opentelemetry.trace import SpanKind
from litellm.integrations.opentelemetry import OpenTelemetry as _OpenTelemetry
from litellm.integrations.opentelemetry import OpenTelemetryConfig as _OpenTelemetryConfig
from litellm.types.integrations.arize import Protocol as _Protocol
Protocol = _Protocol
OpenTelemetryConfig = _OpenTelemetryConfig
Span = Union[_Span, Any]
OpenTelemetry = _OpenTelemetry
else:
Protocol = Any
OpenTelemetryConfig = Any
Span = Any
TracerProvider = Any
SpanKind = Any
# Import OpenTelemetry at runtime
try:
from litellm.integrations.opentelemetry import OpenTelemetry
except ImportError:
OpenTelemetry = None # type: ignore
ARIZE_HOSTED_PHOENIX_ENDPOINT = "https://otlp.arize.com/v1/traces"
class ArizePhoenixLogger(OpenTelemetry):
class ArizePhoenixLogger(OpenTelemetry): # type: ignore
"""
Arize Phoenix logger that sends traces to a Phoenix endpoint.
Creates its own dedicated TracerProvider so it can coexist with the
generic ``otel`` callback (or any other OTEL-based integration) without
fighting over the global ``opentelemetry.trace`` TracerProvider singleton.
"""
def _init_tracing(self, tracer_provider):
"""
Override to always create a *private* TracerProvider for Arize Phoenix.
The base ``OpenTelemetry._init_tracing`` falls back to the global
TracerProvider when one already exists. That causes whichever
integration initialises second to silently reuse the first one's
exporter, so spans only reach one destination.
By creating our own provider we guarantee Arize Phoenix always gets
its own exporter pipeline, regardless of initialisation order.
"""
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import SpanKind
if tracer_provider is not None:
# Explicitly supplied (e.g. in tests) — honour it.
self.tracer = tracer_provider.get_tracer("litellm")
self.span_kind = SpanKind
return
# Always create a dedicated provider — never touch the global one.
provider = TracerProvider(resource=self._get_litellm_resource(self.config))
provider.add_span_processor(self._get_span_processor())
self.tracer = provider.get_tracer("litellm")
self.span_kind = SpanKind
verbose_logger.debug(
"ArizePhoenixLogger: Created dedicated TracerProvider "
"(endpoint=%s, exporter=%s)",
self.config.endpoint,
self.config.exporter,
)
def _init_otel_logger_on_litellm_proxy(self):
"""
Override: Arize Phoenix should NOT overwrite the proxy's
``open_telemetry_logger``. That attribute is reserved for the
primary ``otel`` callback which handles proxy-level parent spans.
"""
pass
def set_attributes(self, span: Span, kwargs, response_obj: Optional[Any]):
ArizePhoenixLogger.set_arize_phoenix_attributes(span, kwargs, response_obj)
return
@staticmethod
def set_arize_phoenix_attributes(span: Span, kwargs, response_obj):
from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes import safe_set_attribute
_utils.set_attributes(span, kwargs, response_obj, ArizeOTELAttributes)
# Set project name on the span for all traces to go to custom Phoenix projects
config = ArizePhoenixLogger.get_arize_phoenix_config()
if config.project_name:
from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes import safe_set_attribute
safe_set_attribute(span, "openinference.project.name", config.project_name)
# Dynamic project name: check metadata first, then fall back to env var config
dynamic_project_name = ArizePhoenixLogger._get_dynamic_project_name(kwargs)
if dynamic_project_name:
safe_set_attribute(span, "openinference.project.name", dynamic_project_name)
else:
# Fall back to static config from env var
config = ArizePhoenixLogger.get_arize_phoenix_config()
if config.project_name:
safe_set_attribute(span, "openinference.project.name", config.project_name)
return
@staticmethod
def _get_dynamic_project_name(kwargs) -> Optional[str]:
"""
Retrieve dynamic Phoenix project name from request metadata.
Users can set `metadata.phoenix_project_name` in their request to route
traces to different Phoenix projects dynamically.
"""
standard_logging_payload = kwargs.get("standard_logging_object")
if isinstance(standard_logging_payload, dict):
metadata = standard_logging_payload.get("metadata")
if isinstance(metadata, dict):
project_name = metadata.get("phoenix_project_name")
if project_name:
return str(project_name)
# Also check litellm_params.metadata for SDK usage
litellm_params = kwargs.get("litellm_params")
if isinstance(litellm_params, dict):
metadata = litellm_params.get("metadata") or {}
else:
metadata = {}
if isinstance(metadata, dict):
project_name = metadata.get("phoenix_project_name")
if project_name:
return str(project_name)
return None
def _handle_success(self, kwargs, response_obj, start_time, end_time):
"""
Override to prevent creating duplicate litellm_request spans when a proxy parent span exists.
ArizePhoenixLogger should reuse the proxy parent span instead of creating a new litellm_request span,
to maintain a shallow span hierarchy as expected by Arize Phoenix.
"""
from opentelemetry.trace import Status, StatusCode
from litellm.secret_managers.main import get_secret_bool
from litellm.integrations.opentelemetry import LITELLM_PROXY_REQUEST_SPAN_NAME
verbose_logger.debug(
"ArizePhoenixLogger: Logging kwargs: %s, OTEL config settings=%s",
kwargs,
self.config,
)
ctx, parent_span = self._get_span_context(kwargs)
# ArizePhoenixLogger NEVER creates a litellm_request span when a proxy parent span exists
# This is different from the base OpenTelemetry behavior which respects USE_OTEL_LITELLM_REQUEST_SPAN
should_create_primary_span = parent_span is None or (
parent_span.name != LITELLM_PROXY_REQUEST_SPAN_NAME
and get_secret_bool("USE_OTEL_LITELLM_REQUEST_SPAN")
)
if should_create_primary_span:
# Create a new litellm_request span
span = self._start_primary_span(
kwargs, response_obj, start_time, end_time, ctx
)
# Raw-request sub-span (if enabled) - child of litellm_request span
self._maybe_log_raw_request(
kwargs, response_obj, start_time, end_time, span
)
# Ensure proxy-request parent span is annotated with the actual operation kind
if (
parent_span is not None
and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
):
self.set_attributes(parent_span, kwargs, response_obj)
else:
# Do not create primary span (keep hierarchy shallow when parent exists)
span = None
# Only set attributes if the span is still recording (not closed)
# Note: parent_span is guaranteed to be not None here
if parent_span.is_recording():
parent_span.set_status(Status(StatusCode.OK))
self.set_attributes(parent_span, kwargs, response_obj)
# Raw-request as direct child of parent_span
self._maybe_log_raw_request(
kwargs, response_obj, start_time, end_time, parent_span
)
# 3. Guardrail span
self._create_guardrail_span(kwargs=kwargs, context=ctx)
# 4. Metrics & cost recording
self._record_metrics(kwargs, response_obj, start_time, end_time)
# 5. Semantic logs.
if self.config.enable_events:
log_span = span if span is not None else parent_span
if log_span is not None:
self._emit_semantic_logs(kwargs, response_obj, log_span)
# 6. Do NOT end parent span - it should be managed by its creator
# External spans (from Langfuse, user code, HTTP headers, global context) must not be closed by LiteLLM
# However, proxy-created spans should be closed here
if (
parent_span is not None
and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
):
parent_span.end(end_time=self._to_ns(end_time))
@staticmethod
def get_arize_phoenix_config() -> ArizePhoenixConfig:
"""

30
litellm/integrations/cloudzero/transform.py
View file

@ -103,10 +103,15 @@ class CBFTransformer:
# Use team_alias if available, otherwise team_id, otherwise fallback to 'unknown'
entity_id = str(team_alias) if team_alias else (str(team_id) if team_id else 'unknown')
# Get alias fields if they exist
api_key_alias = row.get('api_key_alias')
organization_alias = row.get('organization_alias')
project_alias = row.get('project_alias')
user_alias = row.get('user_alias')
dimensions = {
'entity_type': CZEntityType.TEAM.value,
'entity_id': entity_id,
'team_id': str(team_id) if team_id else 'unknown',
'team_alias': str(team_alias) if team_alias else 'unknown',
'model': model,
'model_group': str(row.get('model_group', '')),
@ -119,28 +124,37 @@ class CBFTransformer:
'failed_requests': str(row.get('failed_requests', 0)),
'cache_creation_tokens': str(row.get('cache_creation_input_tokens', 0)),
'cache_read_tokens': str(row.get('cache_read_input_tokens', 0)),
'organization_alias': str(organization_alias) if organization_alias else '',
'project_alias': str(project_alias) if project_alias else '',
'user_alias': str(user_alias) if user_alias else '',
}
# Extract CZRN components to populate corresponding CBF columns
czrn_components = self.czrn_generator.extract_components(resource_id)
service_type, provider, region, owner_account_id, resource_type, cloud_local_id = czrn_components
# Build resource/account as concat of api_key_alias and api_key_prefix
resource_account = f"{api_key_alias}|{api_key_hash}" if api_key_alias else api_key_hash
# CloudZero CBF format with proper column names
cbf_record = {
# Required CBF fields
'time/usage_start': usage_date.isoformat() if usage_date else None, # Required: ISO-formatted UTC datetime
'cost/cost': float(row.get('spend', 0.0)), # Required: billed cost
'resource/id': resource_id, # Required when resource tags are present
'resource/id': model, # Send model name
# Usage metrics for token consumption
'usage/amount': total_tokens, # Numeric value of tokens consumed
'usage/units': 'tokens', # Description of token units
# CBF fields that correspond to CZRN components
'resource/service': service_type, # Maps to CZRN service-type (litellm)
'resource/account': owner_account_id, # Maps to CZRN owner-account-id (entity_id)
# CBF fields - updated per LIT-1907
'resource/service': str(row.get('model_group', '')), # Send model_group
'resource/account': resource_account, # Send api_key_alias|api_key_prefix
'resource/region': region, # Maps to CZRN region (cross-region)
'resource/usage_family': resource_type, # Maps to CZRN resource-type (llm-usage)
'resource/usage_family': str(row.get('custom_llm_provider', '')), # Send provider
# Action field
'action/operation': str(team_id) if team_id else '', # Send team_id
# Line item details
'lineitem/type': 'Usage', # Standard usage line item
@ -155,13 +169,11 @@ class CBFTransformer:
if value and value != 'N/A' and value != 'unknown': # Only add meaningful tags
cbf_record[f'resource/tag:{key}'] = str(value)
# Add token breakdown as resource tags for analysis
# Add token breakdown as resource tags for analysis (excluding total_tokens per LIT-1907)
if prompt_tokens > 0:
cbf_record['resource/tag:prompt_tokens'] = str(prompt_tokens)
if completion_tokens > 0:
cbf_record['resource/tag:completion_tokens'] = str(completion_tokens)
if total_tokens > 0:
cbf_record['resource/tag:total_tokens'] = str(total_tokens)
return CBFRecord(cbf_record)

54
litellm/integrations/custom_guardrail.py
View file

@ -268,6 +268,7 @@ class CustomGuardrail(CustomLogger):
"""
Returns the guardrail(s) to be run from the metadata or root
"""
if "guardrails" in data:
return data["guardrails"]
metadata = data.get("litellm_metadata") or data.get("metadata", {})
@ -615,6 +616,7 @@ class CustomGuardrail(CustomLogger):
end_time: Optional[float] = None,
duration: Optional[float] = None,
event_type: Optional[GuardrailEventHooks] = None,
original_inputs: Optional[Dict] = None,
):
"""
Add StandardLoggingGuardrailInformation to the request data
@ -624,6 +626,17 @@ class CustomGuardrail(CustomLogger):
# Convert None to empty dict to satisfy type requirements
guardrail_response = {} if response is None else response
# For apply_guardrail functions in custom_code_guardrail scenario,
# simplify the logged response to "allow", "deny", or "mask"
if original_inputs is not None and isinstance(response, dict):
# Check if inputs were modified by comparing them
if self._inputs_were_modified(original_inputs, response):
guardrail_response = "mask"
else:
guardrail_response = "allow"
verbose_logger.debug(f"Guardrail response: {response}")
self.add_standard_logging_guardrail_information_to_request_data(
guardrail_json_response=guardrail_response,
request_data=request_data,
@ -649,8 +662,14 @@ class CustomGuardrail(CustomLogger):
This gets logged on downsteam Langfuse, DataDog, etc.
"""
# For custom_code_guardrail scenario, log as "deny" instead of full exception
# Check if this is from custom_code_guardrail by checking the class name
guardrail_response: Union[Exception, str] = e
if "CustomCodeGuardrail" in self.__class__.__name__:
guardrail_response = "deny"
self.add_standard_logging_guardrail_information_to_request_data(
guardrail_json_response=e,
guardrail_json_response=guardrail_response,
request_data=request_data,
guardrail_status="guardrail_failed_to_respond",
duration=duration,
@ -660,6 +679,25 @@ class CustomGuardrail(CustomLogger):
)
raise e
def _inputs_were_modified(self, original_inputs: Dict, response: Dict) -> bool:
"""
Compare original inputs with response to determine if content was modified.
Returns True if the inputs were modified (mask scenario), False otherwise (allow scenario).
"""
# Get all keys from both dictionaries
all_keys = set(original_inputs.keys()) | set(response.keys())
# Compare each key's value
for key in all_keys:
original_value = original_inputs.get(key)
response_value = response.get(key)
if original_value != response_value:
return True
# No modifications detected
return False
def mask_content_in_string(
self,
content_string: str,
@ -767,6 +805,12 @@ def log_guardrail_information(func):
self: CustomGuardrail = args[0]
request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {}
event_type = _infer_event_type_from_function_name(func.__name__)
# Store original inputs for comparison (for apply_guardrail functions)
original_inputs = None
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
original_inputs = kwargs.get("inputs")
try:
response = await func(*args, **kwargs)
return self._process_response(
@ -776,6 +820,7 @@ def log_guardrail_information(func):
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
original_inputs=original_inputs,
)
except Exception as e:
return self._process_error(
@ -793,6 +838,12 @@ def log_guardrail_information(func):
self: CustomGuardrail = args[0]
request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {}
event_type = _infer_event_type_from_function_name(func.__name__)
# Store original inputs for comparison (for apply_guardrail functions)
original_inputs = None
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
original_inputs = kwargs.get("inputs")
try:
response = func(*args, **kwargs)
return self._process_response(
@ -800,6 +851,7 @@ def log_guardrail_information(func):
request_data=request_data,
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
original_inputs=original_inputs,
)
except Exception as e:
return self._process_error(

31
litellm/integrations/custom_logger.py
View file

@ -664,6 +664,37 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
return final_response
"""
pass
async def async_should_run_chat_completion_agentic_loop(
self,
response: Any,
model: str,
messages: List[Dict],
tools: Optional[List[Dict]],
stream: bool,
custom_llm_provider: str,
kwargs: Dict,
) -> Tuple[bool, Dict]:
"""
Hook to determine if chat completion agentic loop should be executed.
"""
return False, {}
async def async_run_chat_completion_agentic_loop(
self,
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
optional_params: Dict,
logging_obj: "LiteLLMLoggingObj",
stream: bool,
kwargs: Dict,
) -> Any:
"""
Hook to execute chat completion agentic loop based on context from should_run hook.
"""
pass
# Useful helpers for custom logger classes

Some files were not shown because too many files have changed in this diff Show more