scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

merge: resolve conflicts between main and litellm_oss_staging_03_04_2026

Browse source 
Resolved 14 file conflicts:
- image_edits.md: combined OpenRouter + Black Forest Labs providers
- utils.py: kept staging's message-level cache_control check
- networking.tsx: kept export on 4 tool interfaces
- tool_management_endpoints.py: kept ToolOutputPolicy import
- Accepted main's version for: schema.prisma, a2a_protocol, mcp_server,
  _types.py, auth_checks.py, db_spend_update_writer, endpoints.py,
  spend_tracking_utils, a2a_endpoints, model_prices backup
This commit is contained in:
Chesars 2026-03-10 10:45:04 -03:00
commit a6cb510703
1136 changed files with 56648 additions and 16667 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

118
.circleci/config.yml
View file

@ -69,9 +69,11 @@ jobs:
- run:
name: Install Python
command: |
choco install python --version=3.11.0 -y
choco install python --version=3.11.0 -y --no-progress --force
refreshenv
python --version
environment:
CHOCOLATEY_CONFIRM_ALL: "true"
- run:
name: Install Dependencies
command: |
@ -3689,6 +3691,114 @@ jobs:
- store_test_results:
path: test-results
proxy_e2e_azure_batches_tests:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Docker CLI
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.12
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.12 -y
conda activate myenv
python --version
- run:
name: Install Poetry
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
pip install poetry
- run:
name: Install dockerize
command: |
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=llmproxy \
-e POSTGRES_PASSWORD=dbpassword9090 \
-e POSTGRES_DB=litellm \
-p 5432:5432 \
postgres:15
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Install system dependencies
command: |
sudo apt-get update -y
sudo apt-get install -y libpq-dev
- run:
name: Install Dependencies
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry config virtualenvs.in-project true
poetry install --with dev,proxy-dev --extras "proxy"
poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
- run:
name: Setup litellm-enterprise
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry run pip install --force-reinstall --no-deps -e enterprise/
- run:
name: Generate Prisma client
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry run prisma generate --schema litellm/proxy/schema.prisma
- run:
name: Run Prisma migrations
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
cd litellm/proxy
poetry run prisma migrate deploy --schema schema.prisma
cd ../..
- run:
name: Run Azure Batch E2E Tests
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
export USE_LOCAL_LITELLM=true
export USE_MOCK_MODELS=true
export USE_STATE_TRACKER=true
export LITELLM_LOG=DEBUG
poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
-vv -s -k "test_e2e_managed_batch" \
--tb=short \
--maxfail=3 \
--durations=10 \
--junitxml=test-results/junit.xml
no_output_timeout: 30m
upload-coverage:
docker:
- image: cimg/python:3.9
@ -4458,6 +4568,12 @@ workflows:
only:
- main
- /litellm_.*/
- proxy_e2e_azure_batches_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- llm_translation_testing:
filters:
branches:

24
.github/workflows/publish_enterprise.yml vendored
View file

@ -19,6 +19,7 @@ jobs:
if: github.repository == 'BerriAI/litellm'
permissions:
contents: write
pull-requests: write
defaults:
run:
working-directory: enterprise
@ -56,14 +57,33 @@ jobs:
- name: Build
run: poetry build
- name: Commit version bump
- name: Commit version bump and create PR
id: create-pr
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
cd ..
BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}"
git checkout -b "$BRANCH"
git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
git push
git push origin "$BRANCH" --force
gh pr create \
--title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \
--body "Version bump for litellm-enterprise. Merge to update main." \
--head "$BRANCH" \
--base main \
|| true
PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url')
echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ github.token }}
- name: Enable auto-merge
run: |
gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash
env:
GH_TOKEN: ${{ github.token }}
- name: Publish to PyPI
env:

1
.github/workflows/test-linting.yml vendored
View file

@ -32,7 +32,6 @@ jobs:
run: |
poetry lock
poetry install --with dev
poetry run pip install openai==1.100.1
- name: Run Black formatting
run: |

2
.github/workflows/test-litellm.yml vendored
View file

@ -38,7 +38,7 @@ jobs:
poetry run pip install "google-genai==1.22.0"
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.22"
poetry run pip install "python-multipart>=0.0.20"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |

90
.github/workflows/test-proxy-e2e-azure-batches.yml vendored Normal file
View file

@ -0,0 +1,90 @@
name: Proxy E2E Azure Batches Tests
on:
pull_request:
branches: [main]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
proxy_e2e_azure_batches_tests:
runs-on: ubuntu-latest
timeout-minutes: 30
services:
postgres:
image: postgres:15
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Cache Poetry dependencies
uses: actions/cache@v4
with:
path: |
~/.cache/pypoetry
~/.cache/pip
.venv
key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }}
restore-keys: |
${{ runner.os }}-poetry-e2e-batches-
${{ runner.os }}-poetry-
- name: Install dependencies
run: |
poetry config virtualenvs.in-project true
poetry install --with dev,proxy-dev --extras "proxy"
poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
- name: Setup litellm-enterprise
run: |
poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Generate Prisma client
run: |
poetry run prisma generate --schema litellm/proxy/schema.prisma
- name: Run Prisma migrations
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
run: |
cd litellm/proxy
poetry run prisma migrate deploy --schema schema.prisma
cd ../..
- name: Run Azure Batch E2E Tests
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
USE_LOCAL_LITELLM: "true"
USE_MOCK_MODELS: "true"
USE_STATE_TRACKER: "true"
LITELLM_LOG: DEBUG
run: |
poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
-vv -s -k "test_e2e_managed_batch" \
--tb=short \
--maxfail=3 \
--durations=10

4
AGENTS.md
View file

@ -212,6 +212,8 @@ When opening issues or pull requests, follow these templates:
Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history.
## HELPFUL RESOURCES
- Main documentation: https://docs.litellm.ai/
@ -249,9 +251,11 @@ The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot
See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`.
- The `--timeout` pytest flag is NOT available; don't pass it.
- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
- Black `--check` may report pre-existing formatting issues; this does not block test runs.
- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file.
### Lint

15
CLAUDE.md
View file

@ -114,4 +114,17 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
### Enterprise Features
- Enterprise-specific code in `enterprise/` directory
- Optional features enabled via environment variables
- Separate licensing and authentication for enterprise features
- Separate licensing and authentication for enterprise features
### HTTP Client Cache Safety
- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`.
### Troubleshooting: DB schema out of sync after proxy restart
`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields.
**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue.
**Fix options:**
1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.

13
Dockerfile
View file

@ -49,7 +49,7 @@ USER root
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
# SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
# levels inside its dependency tree. `npm install -g <pkg>` only creates a
# SEPARATE global package, it does NOT replace npm's internal copies.
@ -70,7 +70,15 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
npm cache clean --force
# SECURITY FIX: patch npm's own package.json metadata so scanners see the
# actual installed versions instead of the stale declared dependencies.
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
# Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is
# no longer visible to image scanners. The globally installed npm@latest
# at /usr/local/lib/node_modules/npm/ remains fully functional.
{ apk del --no-cache npm 2>/dev/null || true; }
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -96,6 +104,7 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \

2
ci_cd/security_scans.sh
View file

@ -161,6 +161,8 @@ run_grype_scans() {
"GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
"GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
"CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
"GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
)
# Build JSON array of allowlisted CVE IDs for jq

7
docker/Dockerfile.custom_ui
View file

@ -19,7 +19,7 @@ RUN apt-get update && apt-get upgrade -y \
libgnutls30 \
libc6 && \
apt-get install -y nodejs npm && \
npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -36,7 +36,10 @@ RUN apt-get update && apt-get upgrade -y \
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
npm cache clean --force
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
apt-get purge -y npm
# Copy the UI source into the container
COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard

8
docker/Dockerfile.database
View file

@ -50,7 +50,7 @@ USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -67,7 +67,10 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
npm cache clean --force
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
{ apk del --no-cache npm 2>/dev/null || true; }
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -85,6 +88,7 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \

8
docker/Dockerfile.dev
View file

@ -75,7 +75,7 @@ RUN apt-get update && apt-get upgrade -y \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
&& npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -92,7 +92,10 @@ RUN apt-get update && apt-get upgrade -y \
&& find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done \
&& npm cache clean --force
&& find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
&& npm cache clean --force \
&& apt-get purge -y npm
WORKDIR /app
@ -114,6 +117,7 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \

8
docker/Dockerfile.non_root
View file

@ -106,7 +106,7 @@ RUN for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done \
&& apk upgrade --no-cache nodejs \
&& npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
&& npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -123,7 +123,10 @@ RUN for i in 1 2 3; do \
&& find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done \
&& npm cache clean --force
&& find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
&& npm cache clean --force \
&& { apk del --no-cache npm 2>/dev/null || true; }
# Copy artifacts from builder
COPY --from=builder /app/requirements.txt /app/requirements.txt
@ -169,6 +172,7 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \

97
docs/my-website/blog/gpt_5_4/index.md Normal file
View file

@ -0,0 +1,97 @@
---
slug: gpt_5_4
title: "Day 0 Support: GPT-5.4"
date: 2026-03-05T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "GPT-5.4 model support in LiteLLM"
tags: [openai, gpt-5.4, completion]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM now supports fully GPT-5.4!
## Docker Image
```bash
docker pull ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch
```
## Usage
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gpt-5.4
litellm_params:
model: openai/gpt-5.4
api_key: os.environ/OPENAI_API_KEY
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch \
--config /app/config.yaml
```
**3. Test it**
```bash
curl -X POST "http://0.0.0.0:4000/chat/completions" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gpt-5.4",
"messages": [
{"role": "user", "content": "Write a Python function to check if a number is prime."}
]
}'
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
from litellm import completion
response = completion(
model="openai/gpt-5.4",
messages=[
{"role": "user", "content": "Write a Python function to check if a number is prime."}
],
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
## Notes
- Restart your container to get the cost tracking for this model.
- Use `/responses` for better model performance.
- GPT-5.4 supports reasoning, function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.

1
docs/my-website/docs/a2a.md
View file

@ -20,6 +20,7 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque
| Logging | ✅ |
| Load Balancing | ✅ |
| Streaming | ✅ |
| [Iteration Budgets](a2a_iteration_budgets) | ✅ |
:::tip

252
docs/my-website/docs/a2a_agent_headers.md Normal file
View file

@ -0,0 +1,252 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# A2A Agent Authentication Headers
Forward authentication credentials (Bearer tokens, API keys, etc.) from clients to backend A2A agents.
## Overview
When LiteLLM proxies a request to a backend A2A agent, the agent may require its own authentication headers. There are three ways to supply them:
| Method | Who configures | How it works |
|---|---|---|
| **Static headers** | Admin (UI / API) | Always sent, regardless of client request |
| **Forward client headers** | Admin (UI / API) | Header names to extract from client request and forward |
| **Convention-based** | Client (no admin config) | Client sends `x-a2a-{agent_name}-{header}` — automatically routed |
All three methods can be combined. **Static headers always win** on key conflicts.
---
## Method 1 — Static Headers
Admin-configured headers that are always sent to the backend agent. Use this for server-to-server tokens or internal credentials that clients should never see or override.
<Tabs>
<TabItem value="ui" label="UI">
1. Go to **Agents** in the LiteLLM dashboard.
2. Create or edit an agent.
3. Open the **Authentication Headers** panel.
4. Under **Static Headers**, click **Add Static Header** and fill in the header name and value.
</TabItem>
<TabItem value="api" label="REST API">
```bash
curl -X POST http://localhost:4000/v1/agents \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"agent_name": "my-agent",
"agent_card_params": { ... },
"static_headers": {
"Authorization": "Bearer internal-server-token",
"X-Internal-Service": "litellm-proxy"
}
}'
```
To update an existing agent:
```bash
curl -X PATCH http://localhost:4000/v1/agents/{agent_id} \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"static_headers": {
"Authorization": "Bearer new-token"
}
}'
```
</TabItem>
</Tabs>
**Client call — no special headers needed:**
```bash
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "Content-Type: application/json" \
-d '{
"jsonrpc": "2.0", "id": "1", "method": "message/send",
"params": { "message": { "role": "user", "parts": [{"kind": "text", "text": "Hello"}], "messageId": "msg-1" } }
}'
```
The backend agent receives `Authorization: Bearer internal-server-token` without the client ever knowing the value.
---
## Method 2 — Forward Client Headers
Admin specifies a list of header **names**. When the client sends a request that includes those headers, LiteLLM extracts their values and forwards them to the backend agent. The client controls the values; the admin controls which headers are eligible to be forwarded.
<Tabs>
<TabItem value="ui" label="UI">
1. Go to **Agents** in the LiteLLM dashboard.
2. Create or edit an agent.
3. Open the **Authentication Headers** panel.
4. Under **Forward Client Headers**, type header names and press **Enter** (e.g. `x-api-key`, `Authorization`).
</TabItem>
<TabItem value="api" label="REST API">
```bash
curl -X POST http://localhost:4000/v1/agents \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"agent_name": "my-agent",
"agent_card_params": { ... },
"extra_headers": ["x-api-key", "x-user-token"]
}'
```
</TabItem>
</Tabs>
**Client call — include the forwarded headers:**
```bash
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "x-api-key: user-secret-value" \
-H "Content-Type: application/json" \
-d '{ ... }'
```
The backend agent receives `x-api-key: user-secret-value`.
:::note
Header name matching is **case-insensitive**. If the client sends `X-API-Key` and `extra_headers` lists `x-api-key`, they match.
:::
---
## Method 3 — Convention-Based Forwarding
Clients can forward headers to a specific agent without any admin pre-configuration by using the naming convention:
```
x-a2a-{agent_name_or_id}-{header_name}: value
```
LiteLLM parses these headers automatically and routes them to the matching agent only.
**Examples:**
| Client header sent | Agent name/ID | Forwarded as |
|---|---|---|
| `x-a2a-my-agent-authorization: Bearer tok` | `my-agent` | `authorization: Bearer tok` |
| `x-a2a-my-agent-x-api-key: secret` | `my-agent` | `x-api-key: secret` |
| `x-a2a-abc123-authorization: Bearer tok` | agent ID `abc123` | `authorization: Bearer tok` |
```bash
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "x-a2a-my-agent-authorization: Bearer agent-specific-token" \
-H "Content-Type: application/json" \
-d '{ ... }'
```
The `x-a2a-other-agent-authorization` header sent in the same request is **not** forwarded to `my-agent` — it is silently ignored.
:::tip Matches both agent name and agent ID
Both the human-readable name (e.g. `my-agent`) and the UUID (e.g. `abc123-...`) are valid. Use whichever is convenient for the client.
:::
---
## Merge Precedence
When multiple methods supply the same header name, **static headers win**:
```
dynamic (forwarded/convention) → merged ← static (overlays, wins)
```
Example:
| Source | `Authorization` value |
|---|---|
| Client sends (via `extra_headers` or convention) | `Bearer client-token` |
| Admin-configured `static_headers` | `Bearer server-token` |
| **What the backend agent receives** | **`Bearer server-token`** |
This ensures admin-controlled credentials cannot be overridden by client requests.
---
## Combining All Three Methods
```bash
# Register agent with static + forwarded headers
curl -X POST http://localhost:4000/v1/agents \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"agent_name": "my-agent",
"agent_card_params": { ... },
"static_headers": {
"X-Internal-Token": "secret123"
},
"extra_headers": ["x-user-id"]
}'
# Client call using all three mechanisms
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "x-user-id: user-42" \
-H "x-a2a-my-agent-x-request-id: req-abc" \
-H "Content-Type: application/json" \
-d '{ ... }'
```
The backend agent receives:
```
X-Internal-Token: secret123 ← static header (always)
x-user-id: user-42 ← forwarded (in extra_headers)
x-request-id: req-abc ← convention-based (x-a2a-my-agent-*)
X-LiteLLM-Trace-Id: <uuid> ← LiteLLM internal
X-LiteLLM-Agent-Id: <agent-id> ← LiteLLM internal
```
---
## Header Isolation
Each agent invocation uses an isolated HTTP connection. Headers configured for agent A are **never** sent to agent B, even if both agents are running and receiving requests simultaneously.
---
## API Reference
### `POST /v1/agents` / `PATCH /v1/agents/{agent_id}`
| Field | Type | Description |
|---|---|---|
| `static_headers` | `object` | `{"Header-Name": "value"}` — always forwarded |
| `extra_headers` | `string[]` | Header names to extract from client request and forward |
### Agent Response
Both fields are returned in `GET /v1/agents` and `GET /v1/agents/{agent_id}`:
```json
{
"agent_id": "...",
"agent_name": "my-agent",
"static_headers": { "X-Internal-Token": "secret123" },
"extra_headers": ["x-user-id"],
...
}
```
:::caution
`static_headers` values are stored in the database and returned by the API. Treat them as you would any credential — do not store sensitive long-lived tokens here if your API is publicly accessible. Consider using short-lived tokens or environment-injected secrets instead.
:::

188
docs/my-website/docs/a2a_iteration_budgets.md Normal file
View file

@ -0,0 +1,188 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Agent Iteration Budgets
Control runaway costs from agentic loops with per-session iteration and budget caps.
## Overview
When agents run agentic loops, they can make unbounded LLM calls, causing unexpected costs. LiteLLM provides two controls:
| Control | Description |
|---------|-------------|
| **Max Iterations** | Hard cap on the number of LLM calls per session |
| **Max Budget Per Session** | Dollar cap per session (identified by `x-litellm-trace-id`) |
Both controls require a `session_id` (sent via `x-litellm-trace-id` header or `metadata.session_id`) to track calls within a session.
## Trace-ID Enforcement
LiteLLM supports two independent trace-id flags, configured in `litellm_params` on the agent:
| Flag | Description |
|------|-------------|
| `require_trace_id_on_calls_to_agent` | Requires callers invoking this agent to include `x-litellm-trace-id`. Use when the agent should only be called as a sub-agent with a trace context. Returns **400** if missing. |
| `require_trace_id_on_calls_by_agent` | Requires all LLM/MCP calls made **by** this agent (via its virtual key) to include `x-litellm-trace-id`. This is what enables `max_iterations` and `max_budget_per_session` tracking. Returns **400** if missing. |
## Configuring via UI
When creating an agent in the LiteLLM Admin UI:
1. Navigate to the **Agents** tab and click **Add Agent**
2. In the **Agent Settings** step, expand the **Tracing** section
3. Toggle **Require x-litellm-trace-id on calls BY this agent** to enable session tracking
4. Set **Max Iterations** to cap the number of LLM calls per session
5. Set **Max Budget Per Session ($)** to cap spend per session
The trace-id flags are stored on the agent's `litellm_params`. Budget controls (`max_iterations`, `max_budget_per_session`) are stored in the virtual key's metadata.
## Configuring via API
Set trace-id enforcement on the agent itself:
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent with budget controls",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"litellm_params": {
"require_trace_id_on_calls_to_agent": true,
"require_trace_id_on_calls_by_agent": true
}
}'
```
Budget controls are set on the agent's `litellm_params` (not on individual keys), so they apply across all keys for the agent:
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent with budget controls",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"litellm_params": {
"require_trace_id_on_calls_by_agent": true,
"max_iterations": 25,
"max_budget_per_session": 5.00
}
}'
```
## How It Works
### Session Tracking
Callers identify their session by including a `session_id` in one of these ways:
- **Header**: `x-litellm-trace-id: my-session-123`
- **Metadata**: `{"metadata": {"session_id": "my-session-123"}}`
### Max Iterations
When `max_iterations` is set in agent `litellm_params`:
- Each LLM call for a session increments a counter
- When the counter exceeds `max_iterations`, the request receives a **429 Too Many Requests**
- Counters expire after 1 hour by default (configurable via `LITELLM_MAX_ITERATIONS_TTL` env var)
### Max Budget Per Session
When `max_budget_per_session` is set in agent `litellm_params`:
- After each successful LLM call, the response cost is accumulated for the session
- Before each call, the accumulated spend is checked against the budget
- When spend exceeds the budget, the request receives a **429 Too Many Requests**
- Session spend counters expire after 1 hour by default (configurable via `LITELLM_MAX_BUDGET_PER_SESSION_TTL` env var)
## Example
Create an agent with max 25 iterations and a $5 budget cap:
<Tabs>
<TabItem value="ui" label="Via UI">
1. Go to **Agents** → **Add Agent**
2. Configure your agent (name, model, etc.)
3. In **Agent Settings**, expand the **Tracing** section
4. Toggle on **Require x-litellm-trace-id on calls BY this agent**
5. Set **Max Iterations** to `25`
6. Set **Max Budget Per Session** to `5.00`
7. Proceed to create a new key for the agent
8. Click **Create Agent**
</TabItem>
<TabItem value="api" label="Via API">
```bash
# 1. Create the agent with trace-id enforcement
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent with budget controls",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"litellm_params": {
"require_trace_id_on_calls_by_agent": true
}
}'
# 2. Create a key for the agent
curl -X POST 'http://localhost:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_id": "<agent_id_from_step_1>",
"key_alias": "my-research-agent-key"
}'
```
</TabItem>
</Tabs>
### Making Calls with Session Tracking
```bash
curl -X POST 'http://localhost:4000/chat/completions' \
-H 'Authorization: Bearer sk-agent-key-xxx' \
-H 'x-litellm-trace-id: session-abc-123' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
After 25 calls or $5 spent within this session, subsequent requests will receive:
```json
{
"error": {
"message": "Session budget exceeded for session session-abc-123. Current spend: $5.0032, max_budget_per_session: $5.00.",
"type": "budget_exceeded",
"code": 429
}
}
```
## Environment Variables
| Variable | Default | Description |
|----------|---------|-------------|
| `LITELLM_MAX_ITERATIONS_TTL` | `3600` (1 hour) | TTL in seconds for session iteration counters |
| `LITELLM_MAX_BUDGET_PER_SESSION_TTL` | `3600` (1 hour) | TTL in seconds for session budget counters |

120
docs/my-website/docs/anthropic_unified/messages_to_responses_mapping.md Normal file
View file

@ -0,0 +1,120 @@
# v1/messages → /responses Parameter Mapping
When you send a request to `/v1/messages` targeting an OpenAI or Azure model, LiteLLM internally routes it through the OpenAI Responses API. This page documents exactly how every parameter gets translated in both directions.
The transformation lives in `litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py`.
## Request: Anthropic → Responses API
### Top-level parameters
| Anthropic (`/v1/messages`) | Responses API | Notes |
|---|---|---|
| `model` | `model` | Passed through as-is |
| `messages` | `input` | Structurally transformed — see the messages section below |
| `system` (string) | `instructions` | Passed as a plain string |
| `system` (list of content blocks) | `instructions` | Text blocks are joined with `\n`; non-text blocks are ignored |
| `max_tokens` | `max_output_tokens` | Renamed |
| `temperature` | `temperature` | Passed through as-is |
| `top_p` | `top_p` | Passed through as-is |
| `tools` | `tools` | Format-translated — see the tools section below |
| `tool_choice` | `tool_choice` | Type-remapped — see the tool_choice section below |
| `thinking` | `reasoning` | Budget tokens mapped to effort level — see the thinking section below |
| `output_format` or `output_config.format` | `text` | Wrapped as `{"format": {"type": "json_schema", "name": "structured_output", "schema": ..., "strict": true}}` |
| `context_management` | `context_management` | Converted from Anthropic dict to OpenAI array format — see the context_management section below |
| `metadata.user_id` | `user` | Extracted from the metadata object and truncated to 64 characters |
| `stop_sequences` | ❌ Not mapped | Dropped silently |
| `top_k` | ❌ Not mapped | Dropped silently |
| `speed` | ❌ Not mapped | Only used to set Anthropic beta headers on the native path |
### How messages get converted
Each Anthropic message is expanded into one or more Responses API input items. The key difference is that `tool_result` and `tool_use` blocks become **top-level items** in the input array rather than being nested inside a message.
| Anthropic message | Responses API input item |
|---|---|
| `user` role, string content | `{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "..."}]}` |
| `user` role, `{"type": "text"}` block | `{"type": "input_text", "text": "..."}` inside a user message |
| `user` role, `{"type": "image", "source": {"type": "base64"}}` | `{"type": "input_image", "image_url": "data:<media_type>;base64,<data>"}` inside a user message |
| `user` role, `{"type": "image", "source": {"type": "url"}}` | `{"type": "input_image", "image_url": "<url>"}` inside a user message |
| `user` role, `{"type": "tool_result"}` block | Top-level `{"type": "function_call_output", "call_id": "...", "output": "..."}` — pulled out of the message entirely |
| `assistant` role, string content | `{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "..."}]}` |
| `assistant` role, `{"type": "text"}` block | `{"type": "output_text", "text": "..."}` inside an assistant message |
| `assistant` role, `{"type": "tool_use"}` block | Top-level `{"type": "function_call", "call_id": "<id>", "name": "...", "arguments": "<JSON string>"}` — pulled out of the message entirely |
| `assistant` role, `{"type": "thinking"}` block | `{"type": "output_text", "text": "<thinking text>"}` inside an assistant message |
### tools
| Anthropic tool | Responses API tool |
|---|---|
| Any tool where `type` starts with `"web_search"` or `name == "web_search"` | `{"type": "web_search_preview"}` |
| All other tools | `{"type": "function", "name": "...", "description": "...", "parameters": <input_schema>}` |
### tool_choice
| Anthropic `tool_choice.type` | Responses API `tool_choice` |
|---|---|
| `"auto"` | `{"type": "auto"}` |
| `"any"` | `{"type": "required"}` |
| `"tool"` | `{"type": "function", "name": "<tool name>"}` |
### thinking → reasoning
The `budget_tokens` value is mapped to a string effort level. `summary` is always set to `"detailed"`.
| `thinking.budget_tokens` | `reasoning.effort` |
|---|---|
| >= 10000 | `"high"` |
| >= 5000 | `"medium"` |
| >= 2000 | `"low"` |
| < 2000 | `"minimal"` |
If `thinking.type` is anything other than `"enabled"`, the `reasoning` field is not sent at all.
### context_management
Anthropic uses a nested dict with an `edits` array. OpenAI uses a flat array of compaction objects.
```
Anthropic input:
{
"edits": [
{
"type": "compact_20260112",
"trigger": {"type": "input_tokens", "value": 150000}
}
]
}
Responses API output:
[
{"type": "compaction", "compact_threshold": 150000}
]
```
## Response: Responses API → Anthropic
When the Responses API reply comes back, LiteLLM converts it into an Anthropic `AnthropicMessagesResponse`.
| Responses API field | Anthropic response field | Notes |
|---|---|---|
| `response.id` | `id` | |
| `response.model` | `model` | Falls back to `"unknown-model"` if missing |
| `ResponseReasoningItem` — `summary[*].text` | `content` block `{"type": "thinking", "thinking": "..."}` | Each non-empty summary text becomes a thinking block |
| `ResponseOutputMessage` — `content[*]` where `type == "output_text"` | `content` block `{"type": "text", "text": "..."}` | |
| `ResponseFunctionToolCall` — `{call_id, name, arguments}` | `content` block `{"type": "tool_use", "id": "...", "name": "...", "input": {...}}` | `arguments` is JSON-parsed back into a dict |
| Any `function_call` present in output | `stop_reason: "tool_use"` | |
| `response.status == "incomplete"` | `stop_reason: "max_tokens"` | Takes precedence over the default |
| Everything else | `stop_reason: "end_turn"` | Default |
| `response.usage.input_tokens` | `usage.input_tokens` | |
| `response.usage.output_tokens` | `usage.output_tokens` | |
| *(hardcoded)* | `type: "message"` | Always set |
| *(hardcoded)* | `role: "assistant"` | Always set |
| *(hardcoded)* | `stop_sequence: null` | Always null on this path |

71
docs/my-website/docs/image_edits.md
View file

@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| Supported operations | Create image edits | Single and multiple images supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
| Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **Stability AI**, **AWS Bedrock (Stability)**, **Black Forest Labs** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. Stability AI and Bedrock Stability support various image editing operations. Black Forest Labs supports FLUX Kontext models. |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **OpenRouter**, **Stability AI**, **AWS Bedrock (Stability)**, **Black Forest Labs** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. OpenRouter routes image edits through chat completions. Stability AI and Bedrock Stability support various image editing operations. Black Forest Labs supports FLUX Kontext models. |
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
@ -301,6 +301,47 @@ response = litellm.image_edit(
print(response)
```
</TabItem>
<TabItem value="openrouter" label="OpenRouter">
#### Basic Image Edit
```python showLineNumbers title="OpenRouter Image Edit"
import os
from litellm import image_edit
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=open("original_image.png", "rb"),
prompt="Add aurora borealis to the night sky",
)
print(response)
```
#### Multiple Images Edit
```python showLineNumbers title="OpenRouter Multiple Images Edit"
import os
from litellm import image_edit
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=[
open("scene.png", "rb"),
open("style_reference.png", "rb"),
],
prompt="Blend the reference style into the scene",
size="1536x1024", # mapped to aspect_ratio 3:2
quality="high", # mapped to image_size 4K
)
print(response)
```
</TabItem>
</Tabs>
@ -484,6 +525,34 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-F "size=1024x1024"
```
</TabItem>
<TabItem value="openrouter" label="OpenRouter">
1. Add the OpenRouter image edit model to your `config.yaml`:
```yaml showLineNumbers title="OpenRouter Proxy Configuration"
model_list:
- model_name: openrouter-image-edit
litellm_params:
model: openrouter/google/gemini-2.5-flash-image
api_key: os.environ/OPENROUTER_API_KEY
```
2. Start the LiteLLM proxy server:
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
```
3. Make an image edit request:
```bash showLineNumbers title="OpenRouter Proxy Image Edit"
curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-F "model=openrouter-image-edit" \
-F "image=@original_image.png" \
-F "prompt=Make the sky a vibrant purple sunset" \
-F "size=1024x1024"
```
</TabItem>
</Tabs>

227
docs/my-website/docs/mcp.md
View file

@ -336,175 +336,9 @@ litellm_settings:
## Converting OpenAPI Specs to MCP Servers
LiteLLM can automatically convert OpenAPI specifications into MCP servers, allowing you to expose any REST API as MCP tools. This is useful when you have existing APIs with OpenAPI/Swagger documentation and want to make them available as MCP tools.
LiteLLM can convert OpenAPI specifications into MCP servers, exposing any REST API as MCP tools without writing custom server code.
**Benefits:**
- **Rapid Integration**: Convert existing APIs to MCP tools without writing custom MCP server code
- **Automatic Tool Generation**: LiteLLM automatically generates MCP tools from your OpenAPI spec
- **Unified Interface**: Use the same MCP interface for both native MCP servers and OpenAPI-based APIs
- **Easy Testing**: Test and iterate on API integrations quickly
**Configuration:**
Add your OpenAPI-based MCP server to your `config.yaml`:
```yaml title="config.yaml - OpenAPI to MCP" showLineNumbers
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-xxxxxxx
mcp_servers:
# OpenAPI Spec Example - Petstore API
petstore_mcp:
url: "https://petstore.swagger.io/v2"
spec_path: "/path/to/openapi.json"
auth_type: "none"
# OpenAPI Spec with API Key Authentication
my_api_mcp:
url: "http://0.0.0.0:8090"
spec_path: "/path/to/openapi.json"
auth_type: "api_key"
auth_value: "your-api-key-here"
# OpenAPI Spec with Bearer Token
secured_api_mcp:
url: "https://api.example.com"
spec_path: "/path/to/openapi.json"
auth_type: "bearer_token"
auth_value: "your-bearer-token"
```
**Configuration Parameters:**
| Parameter | Required | Description |
|-----------|----------|-------------|
| `url` | Yes | The base URL of your API endpoint |
| `spec_path` | Yes | Path or URL to your OpenAPI specification file (JSON or YAML) |
| `auth_type` | No | Authentication type: `none`, `api_key`, `bearer_token`, `basic`, `authorization` |
| `auth_value` | No | Authentication value (required if `auth_type` is set) |
| `authorization_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `token_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `registration_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `scopes` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM uses the scopes advertised by the server. |
| `description` | No | Optional description for the MCP server |
| `allowed_tools` | No | List of specific tools to allow (see [MCP Tool Filtering](#mcp-tool-filtering)) |
| `disallowed_tools` | No | List of specific tools to block (see [MCP Tool Filtering](#mcp-tool-filtering)) |
### Usage Example
Once configured, you can use the OpenAPI-based MCP server just like any other MCP server:
<Tabs>
<TabItem value="fastmcp" label="Python FastMCP">
```python title="Using OpenAPI-based MCP Server" showLineNumbers
from fastmcp import Client
import asyncio
# Standard MCP configuration
config = {
"mcpServers": {
"petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
}
}
}
# Create a client that connects to the server
client = Client(config)
async def main():
async with client:
# List available tools generated from OpenAPI spec
tools = await client.list_tools()
print(f"Available tools: {[tool.name for tool in tools]}")
# Example: Get a pet by ID (from Petstore API)
response = await client.call_tool(
name="getpetbyid",
arguments={"petId": "1"}
)
print(f"Response:\n{response}\n")
# Example: Find pets by status
response = await client.call_tool(
name="findpetsbystatus",
arguments={"status": "available"}
)
print(f"Response:\n{response}\n")
if __name__ == "__main__":
asyncio.run(main())
```
</TabItem>
<TabItem value="cursor" label="Cursor IDE">
```json title="Cursor MCP Configuration for OpenAPI Server" showLineNumbers
{
"mcpServers": {
"Petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY"
}
}
}
}
```
</TabItem>
<TabItem value="openai" label="OpenAI Responses API">
```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
curl --location 'https://api.openai.com/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer $OPENAI_API_KEY" \
--data '{
"model": "gpt-4o",
"tools": [
{
"type": "mcp",
"server_label": "petstore",
"server_url": "http://localhost:4000/petstore_mcp/mcp",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
}
}
],
"input": "Find all available pets in the petstore",
"tool_choice": "required"
}'
```
</TabItem>
</Tabs>
**How It Works**
1. **Spec Loading**: LiteLLM loads your OpenAPI specification from the provided `spec_path`
2. **Tool Generation**: Each API endpoint in the spec becomes an MCP tool
3. **Parameter Mapping**: OpenAPI parameters are automatically mapped to MCP tool parameters
4. **Request Handling**: When a tool is called, LiteLLM converts the MCP request to the appropriate HTTP request
5. **Response Translation**: API responses are converted back to MCP format
**OpenAPI Spec Requirements**
Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
- **Supported versions**: OpenAPI 3.0.x, OpenAPI 3.1.x, Swagger 2.0
- **Required fields**: `paths`, `info` sections should be properly defined
- **Operation IDs**: Each operation should have a unique `operationId` (this becomes the tool name)
- **Parameters**: Request parameters should be properly documented with types and descriptions
See the **[MCP from OpenAPI Specs guide](./mcp_openapi.md)** for full setup, usage examples, and how to override tool names and descriptions.
## MCP OAuth
@ -870,6 +704,63 @@ asyncio.run(main())
[Learn more about customer management →](./proxy/customers)
## Calling the Proxy's /v1/responses Endpoint
When calling your LiteLLM Proxy's `/v1/responses` endpoint to use MCP tools, **always use `server_url: "litellm_proxy"`** in the tools array. This tells the proxy to use its configured MCP servers.
:::important Do not use the full proxy URL
Using `server_url: "https://your-proxy.com/mcp"` is incorrect when the request is already going to the proxy. The proxy needs the literal value `litellm_proxy` to route to its configured MCP servers.
:::
```bash title="Correct: Using litellm_proxy" showLineNumbers
curl --location 'https://your-proxy.com/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer $LITELLM_API_KEY" \
--data '{
"model": "gpt-4",
"tools": [
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never"
}
],
"input": "Run available tools",
"tool_choice": "required"
}'
```
### Sending Custom Headers to MCP Servers
To pass custom headers (e.g., API keys, auth tokens) to specific MCP servers, use either:
**Option 1: Request headers** – Add `x-mcp-{server_alias}-{header_name}` to your request headers. The proxy forwards these to the matching MCP server.
```bash
# Send Authorization header to the "weather2" MCP server
--header 'x-mcp-weather2-authorization: Bearer your-token'
# Send custom header to the "github" MCP server
--header 'x-mcp-github-x-api-key: your-api-key'
```
**Option 2: Headers in tool config** – Include a `headers` object in the tool definition. These are merged with request headers.
```json
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
"x-mcp-servers": "Zapier_MCP,dev-group",
"x-mcp-weather2-authorization": "Bearer your-weather-api-token"
}
}
```
## Using your MCP with client side credentials
Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.

8
docs/my-website/docs/mcp_control.md
View file

@ -323,7 +323,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
{
"type": "mcp",
"server_label": "litellm",
"server_url": "<your-litellm-proxy-base-url>/dev_group/mcp",
"server_url": "litellm_proxy",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
@ -335,7 +335,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
}'
```
This example uses URL namespacing to access all servers in the "dev_group" access group.
This example uses the `x-mcp-servers` header to access all servers in the "dev_group" access group. Use `server_url: "litellm_proxy"` when calling the proxy's `/v1/responses` endpoint—do not use the full proxy URL.
</TabItem>
@ -423,7 +423,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
{
"type": "mcp",
"server_label": "litellm",
"server_url": "<your-litellm-proxy-base-url>/mcp/",
"server_url": "litellm_proxy",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
@ -436,7 +436,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
}'
```
This configuration restricts the request to only use tools from the specified MCP servers.
This configuration restricts the request to only use tools from the specified MCP servers. Use `server_url: "litellm_proxy"` when calling the proxy's `/v1/responses` endpoint.
</TabItem>

226
docs/my-website/docs/mcp_openapi.md Normal file
View file

@ -0,0 +1,226 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# MCP from OpenAPI Specs
LiteLLM can convert any OpenAPI/Swagger spec into an MCP server — no custom MCP server code required.
## Step 1 — Add the MCP Server
Add your OpenAPI-based server in `config.yaml`:
```yaml title="config.yaml" showLineNumbers
mcp_servers:
petstore_mcp:
url: "https://petstore.swagger.io/v2"
spec_path: "/path/to/openapi.json"
auth_type: "none"
my_api_mcp:
url: "http://0.0.0.0:8090"
spec_path: "/path/to/openapi.json"
auth_type: "api_key"
auth_value: "your-api-key-here"
secured_api_mcp:
url: "https://api.example.com"
spec_path: "/path/to/openapi.json"
auth_type: "bearer_token"
auth_value: "your-bearer-token"
```
Or from the UI: go to **MCP Servers → Add New MCP Server**, fill in the URL and spec path, and LiteLLM will fetch the spec and load all endpoints as tools.
**Configuration parameters:**
| Parameter | Required | Description |
|-----------|----------|-------------|
| `url` | Yes | Base URL of your API |
| `spec_path` | Yes | Path or URL to your OpenAPI spec (JSON or YAML) |
| `auth_type` | No | `none`, `api_key`, `bearer_token`, `basic`, `authorization`, `oauth2` |
| `auth_value` | No | Auth value (required if `auth_type` is set) |
| `description` | No | Optional description |
| `allowed_tools` | No | Allowlist of specific tools |
| `disallowed_tools` | No | Blocklist of specific tools |
**Supported spec versions:** OpenAPI 3.0.x, 3.1.x, Swagger 2.0. Each operation's `operationId` becomes the tool name — make sure they're unique.
Once tools are loaded, you'll see them in the Tool Configuration section:
<Image
img={require('../img/mcp_openapi_tools_loaded.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
## Step 2 — Optionally Override Tool Names and Descriptions
By default, tool names and descriptions come from the `operationId` and description fields in your spec. You can rename or rewrite them so MCP clients see something cleaner — without touching the upstream spec.
### From the UI
Each tool card has a pencil icon. Click it to open the inline editor:
<Image
img={require('../img/mcp_openapi_tool_edit_panel.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
- **Display Name** — overrides the name MCP clients see
- **Description** — overrides the description MCP clients see
- Leave a field blank to keep the original from the spec
After setting overrides, a purple **Custom name** badge appears on the tool card:
<Image
img={require('../img/mcp_openapi_custom_name_badge.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
### From the API
Pass `tool_name_to_display_name` and `tool_name_to_description` in the create or update request:
```bash title="Create server with tool name overrides" showLineNumbers
curl -X POST http://localhost:4000/v1/mcp/server \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"name": "petstore_mcp",
"url": "https://petstore.swagger.io/v2",
"spec_path": "/path/to/openapi.json",
"tool_name_to_display_name": {
"getPetById": "Get Pet",
"findPetsByStatus": "List Available Pets"
},
"tool_name_to_description": {
"getPetById": "Look up a pet by its ID",
"findPetsByStatus": "Returns all pets matching a given status (available, pending, sold)"
}
}'
```
```bash title="Update overrides on an existing server" showLineNumbers
curl -X PUT http://localhost:4000/v1/mcp/server/{server_id} \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"tool_name_to_display_name": {
"getPetById": "Get Pet"
},
"tool_name_to_description": {
"getPetById": "Look up a pet by its ID"
}
}'
```
The map key is the **original `operationId`** from the spec — not the prefixed tool name. LiteLLM strips the server prefix before doing the lookup.
For example, if your server is `petstore_mcp`, the tool is exposed as `petstore_mcp-getPetById`. The map key is still `getPetById`.
**Before and after:**
```
# Without overrides
Tool: "petstore_mcp-getPetById"
Description: "Returns a single pet"
Tool: "petstore_mcp-findPetsByStatus"
Description: "Finds Pets by status"
# After overrides
Tool: "Get Pet"
Description: "Look up a pet by its ID"
Tool: "List Available Pets"
Description: "Returns all pets matching a given status (available, pending, sold)"
```
## Using the Server
<Tabs>
<TabItem value="fastmcp" label="Python FastMCP">
```python title="Using OpenAPI-based MCP Server" showLineNumbers
from fastmcp import Client
import asyncio
config = {
"mcpServers": {
"petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
}
}
}
client = Client(config)
async def main():
async with client:
tools = await client.list_tools()
print(f"Available tools: {[tool.name for tool in tools]}")
response = await client.call_tool(
name="Get Pet", # overridden name
arguments={"petId": "1"}
)
print(f"Response: {response}")
if __name__ == "__main__":
asyncio.run(main())
```
</TabItem>
<TabItem value="cursor" label="Cursor IDE">
```json title="Cursor MCP Configuration" showLineNumbers
{
"mcpServers": {
"Petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY"
}
}
}
}
```
</TabItem>
<TabItem value="openai" label="OpenAI Responses API">
```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
curl --location 'https://api.openai.com/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer $OPENAI_API_KEY" \
--data '{
"model": "gpt-4o",
"tools": [
{
"type": "mcp",
"server_label": "petstore",
"server_url": "http://localhost:4000/petstore_mcp/mcp",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
}
}
],
"input": "Find all available pets",
"tool_choice": "required"
}'
```
</TabItem>
</Tabs>

72
docs/my-website/docs/providers/azure_ai/azure_model_router.md
View file

@ -2,6 +2,32 @@
Azure Model Router is a feature in Azure AI Foundry that automatically routes your requests to the best available model based on your requirements. This allows you to use a single endpoint that intelligently selects the optimal model for each request.
## Quick Start
**Model pattern**: `azure_ai/model_router/<deployment-name>`
```python
import litellm
response = litellm.completion(
model="azure_ai/model_router/model-router", # Replace with your deployment name
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key="your-api-key",
)
```
**Proxy config** (`config.yaml`):
```yaml
model_list:
- model_name: model-router
litellm_params:
model: azure_ai/model_router/model-router
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/deployments/model-router/chat/completions?api-version=2025-01-01-preview
api_key: your-api-key
```
## Key Features
- **Automatic Model Selection**: Azure Model Router dynamically selects the best model for your request
@ -229,19 +255,51 @@ Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`), plus a fl
## Cost Tracking
LiteLLM automatically handles cost tracking for Azure Model Router by:
LiteLLM automatically handles cost tracking for Azure Model Router. Understanding how this works helps you interpret spend and debug billing.
1. **Detecting the actual model**: When Azure Model Router routes your request to a specific model (e.g., `gpt-4.1-nano-2025-04-14`), LiteLLM extracts this from the response
2. **Calculating accurate costs**: Costs are calculated based on:
- The actual model used (e.g., `gpt-4.1-nano` token costs)
- Plus a flat infrastructure cost of **$0.14 per million input tokens** for using the Model Router
3. **Streaming support**: Cost tracking works correctly for both streaming and non-streaming requests
### How LiteLLM Calculates Cost
When you use Azure Model Router, LiteLLM computes **two cost components**:
| Component | Description | When Applied |
|-----------|-------------|--------------|
| **Model Cost** | Token-based cost for the actual model that handled the request (e.g., `gpt-5-nano`, `gpt-4.1-nano`) | Always, when Azure returns the model in the response |
| **Router Flat Cost** | $0.14 per million input tokens (Azure AI Foundry infrastructure fee) | When the **request** was made via a model router endpoint |
### Cost Calculation Flow
1. **Request model detection**: LiteLLM records the model you requested (e.g., `azure_ai/model_router/model-router`). If it contains `model_router` or `model-router`, the request is treated as a router request.
2. **Response model extraction**: Azure returns the actual model used in the response (e.g., `gpt-5-nano-2025-08-07`). LiteLLM uses this for the model cost lookup.
3. **Model cost**: LiteLLM looks up the response model in its pricing table and computes cost from prompt tokens and completion tokens.
4. **Router flat cost**: Because the original request was to a model router, LiteLLM adds the flat cost ($0.14 per M input tokens) on top of the model cost.
5. **Total cost**: `Total = Model Cost + Router Flat Cost`
### Configuration Requirements
For cost tracking to work correctly:
- **Use the full pattern**: `azure_ai/model_router/<deployment-name>` (e.g., `azure_ai/model_router/model-router`)
- **Proxy config**: When using the LiteLLM proxy, set `model` in `litellm_params` to the full pattern so the request model is correctly identified as a router
```yaml
# proxy_server_config.yaml
model_list:
- model_name: model-router
litellm_params:
model: azure_ai/model_router/model-router # Required for router cost detection
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/deployments/model-router/chat/completions?api-version=2025-01-01-preview
api_key: your-api-key
```
### Cost Breakdown
When you use Azure Model Router, the total cost includes:
- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-4.1-nano`)
- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-5-nano`, `gpt-4.1-nano`)
- **Router Flat Cost**: $0.14 per million input tokens (Azure AI Foundry infrastructure fee)
### Example Response with Cost

157
docs/my-website/docs/providers/bedrock_mantle.md Normal file
View file

@ -0,0 +1,157 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Amazon Bedrock Mantle
[Amazon Bedrock Mantle](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html) is Amazon Bedrock's distributed inference engine (Project Mantle) that exposes an **OpenAI-compatible API** for Bedrock-hosted models.
Use this provider to call Bedrock Mantle models with accurate **AWS Bedrock pricing** instead of OpenAI pricing.
:::tip
**We support ALL Bedrock Mantle models, just set `model=bedrock_mantle/<model-id>` as a prefix when sending litellm requests**
:::
## API Key
```python
# env variable
os.environ['BEDROCK_MANTLE_API_KEY'] = "your-aws-bedrock-api-key"
# optional: override region (defaults to us-east-1)
os.environ['BEDROCK_MANTLE_REGION'] = "us-east-1" # or use AWS_REGION
```
## Supported Models
| Model | Context Window | Input (per 1M tokens) | Output (per 1M tokens) |
|-------|---------------|----------------------|------------------------|
| `openai.gpt-oss-120b` | 131K | $0.15 | $0.60 |
| `openai.gpt-oss-20b` | 131K | $0.075 | $0.30 |
| `openai.gpt-oss-safeguard-120b` | 131K | $0.15 | $0.60 |
| `openai.gpt-oss-safeguard-20b` | 131K | $0.075 | $0.30 |
## Sample Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key"
response = completion(
model="bedrock_mantle/openai.gpt-oss-120b",
messages=[{"role": "user", "content": "hello from litellm"}],
)
print(response)
```
</TabItem>
<TabItem value="streaming" label="Streaming">
```python
from litellm import completion
import os
os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key"
response = completion(
model="bedrock_mantle/openai.gpt-oss-120b",
messages=[{"role": "user", "content": "hello from litellm"}],
stream=True,
)
for chunk in response:
print(chunk)
```
</TabItem>
<TabItem value="async" label="Async">
```python
import asyncio
from litellm import acompletion
import os
os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key"
async def main():
response = await acompletion(
model="bedrock_mantle/openai.gpt-oss-120b",
messages=[{"role": "user", "content": "hello from litellm"}],
)
print(response)
asyncio.run(main())
```
</TabItem>
</Tabs>
## Region Configuration
The API base URL is `https://bedrock-mantle.{region}.api.aws/v1`. Region is resolved in this order:
1. `BEDROCK_MANTLE_REGION` env var
2. `AWS_REGION` env var
3. Default: `us-east-1`
**Supported regions:** `us-east-1`, `us-east-2`, `us-west-2`, `eu-west-1`, `eu-west-2`, `eu-central-1`, `eu-south-1`, `eu-north-1`, `ap-northeast-1`, `ap-south-1`, `ap-southeast-3`, `sa-east-1`
```python
import os
os.environ['BEDROCK_MANTLE_REGION'] = "eu-west-1"
# or pass api_base directly
response = completion(
model="bedrock_mantle/openai.gpt-oss-120b",
messages=[{"role": "user", "content": "hello"}],
api_base="https://bedrock-mantle.eu-west-1.api.aws/v1",
)
```
## Usage with LiteLLM Proxy
### 1. Set Bedrock Mantle models on config.yaml
```yaml
model_list:
- model_name: gpt-oss-120b
litellm_params:
model: bedrock_mantle/openai.gpt-oss-120b
api_key: os.environ/BEDROCK_MANTLE_API_KEY
# optional region override:
api_base: "https://bedrock-mantle.us-east-1.api.aws/v1"
- model_name: gpt-oss-20b
litellm_params:
model: bedrock_mantle/openai.gpt-oss-20b
api_key: os.environ/BEDROCK_MANTLE_API_KEY
```
### 2. Start the proxy
```shell
litellm --config /path/to/config.yaml
```
### 3. Send a request
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000",
)
response = client.chat.completions.create(
model="gpt-oss-120b",
messages=[{"role": "user", "content": "hello from litellm"}],
)
print(response)
```

36
docs/my-website/docs/providers/chatgpt.md
View file

@ -4,12 +4,12 @@ Use ChatGPT Pro/Max subscription models through LiteLLM with OAuth device flow a
| Property | Details |
|-------|-------|
| Description | ChatGPT subscription access (Codex + GPT-5.2 family) via ChatGPT backend API |
| Description | ChatGPT subscription access (Codex + GPT-5.3/5.4 family) via ChatGPT backend API |
| Provider Route on LiteLLM | `chatgpt/` |
| Supported Endpoints | `/responses`, `/chat/completions` (bridged to Responses for supported models) |
| API Reference | https://chatgpt.com |
ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.2`).
ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.4`).
Notes:
- The ChatGPT subscription backend rejects token limit fields (`max_tokens`, `max_output_tokens`, `max_completion_tokens`) and `metadata`. LiteLLM strips these fields for this provider.
@ -31,7 +31,7 @@ ChatGPT subscription access uses an OAuth device code flow:
import litellm
response = litellm.responses(
model="chatgpt/gpt-5.2-codex",
model="chatgpt/gpt-5.3-codex",
input="Write a Python hello world"
)
@ -44,7 +44,7 @@ print(response)
import litellm
response = litellm.completion(
model="chatgpt/gpt-5.2",
model="chatgpt/gpt-5.4",
messages=[{"role": "user", "content": "Write a Python hello world"}]
)
@ -55,16 +55,36 @@ print(response)
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: chatgpt/gpt-5.2
- model_name: chatgpt/gpt-5.4
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.2
- model_name: chatgpt/gpt-5.2-codex
model: chatgpt/gpt-5.4
- model_name: chatgpt/gpt-5.4-pro
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.2-codex
model: chatgpt/gpt-5.4-pro
- model_name: chatgpt/gpt-5.3-codex
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-codex
- model_name: chatgpt/gpt-5.3-codex-spark
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-codex-spark
- model_name: chatgpt/gpt-5.3-instant
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-instant
- model_name: chatgpt/gpt-5.3-chat-latest
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-chat-latest
```
```bash showLineNumbers title="Start LiteLLM Proxy"

5
docs/my-website/docs/providers/openai.md
View file

@ -191,8 +191,13 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL
| gpt-5.2 | `response = completion(model="gpt-5.2", messages=messages)` |
| gpt-5.2-2025-12-11 | `response = completion(model="gpt-5.2-2025-12-11", messages=messages)` |
| gpt-5.2-chat-latest | `response = completion(model="gpt-5.2-chat-latest", messages=messages)` |
| gpt-5.3-chat-latest | `response = completion(model="gpt-5.3-chat-latest", messages=messages)` |
| gpt-5.4 | `response = completion(model="gpt-5.4", messages=messages)` |
| gpt-5.4-2026-03-05 | `response = completion(model="gpt-5.4-2026-03-05", messages=messages)` |
| gpt-5.2-pro | `response = completion(model="gpt-5.2-pro", messages=messages)` |
| gpt-5.2-pro-2025-12-11 | `response = completion(model="gpt-5.2-pro-2025-12-11", messages=messages)` |
| gpt-5.4-pro | `response = completion(model="gpt-5.4-pro", messages=messages)` |
| gpt-5.4-pro-2026-03-05 | `response = completion(model="gpt-5.4-pro-2026-03-05", messages=messages)` |
| gpt-5.1 | `response = completion(model="gpt-5.1", messages=messages)` |
| gpt-5.1-codex | `response = completion(model="gpt-5.1-codex", messages=messages)` |
| gpt-5.1-codex-mini | `response = completion(model="gpt-5.1-codex-mini", messages=messages)` |

87
docs/my-website/docs/providers/openrouter.md
View file

@ -210,3 +210,90 @@ response = image_generation(
# Cost is available in the response metadata
print(f"Request cost: ${response._hidden_params['additional_headers']['llm_provider-x-litellm-response-cost']}")
```
## Image Edit
OpenRouter supports image editing through select models like Google Gemini image models. LiteLLM routes image edit requests to OpenRouter's chat completions endpoint with the source image sent as a base64 data URL and `modalities: ["image", "text"]`.
### Supported Models
| Model | Description |
|-------|-------------|
| `openrouter/google/gemini-2.5-flash-image` | Gemini 2.5 Flash with image editing |
See all available image models on [OpenRouter's model list](https://openrouter.ai/models?modality=image).
### Supported Parameters
| Parameter | OpenRouter Mapping | Notes |
|-----------|--------------------|-------|
| `size` | `image_config.aspect_ratio` | `1024x1024` → `1:1`, `1536x1024` → `3:2`, `1024x1536` → `2:3`, `1792x1024` → `16:9`, `1024x1792` → `9:16` |
| `quality` | `image_config.image_size` | `low`/`standard` → `1K`, `medium` → `2K`, `high`/`hd` → `4K` |
| `n` | `n` | Number of images |
:::note
`quality=high` (4K) is only supported by `google/gemini-3-pro-image-preview` and `google/gemini-3.1-flash-image-preview`. The `google/gemini-2.5-flash-image` model supports up to `medium` (2K).
:::
### Usage
```python
from litellm import image_edit
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
# Basic image edit
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=open("original_image.png", "rb"),
prompt="Make the sky a vibrant purple sunset",
)
print(response)
```
### Advanced Usage with Parameters
```python
from litellm import image_edit
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
# Edit with size and quality parameters
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=open("photo.png", "rb"),
prompt="Add northern lights to the sky",
size="1536x1024", # Maps to aspect_ratio 3:2
quality="high", # Maps to image_size 4K
)
# Access the edited image
image_data = response.data[0]
if image_data.b64_json:
import base64
with open("edited.png", "wb") as f:
f.write(base64.b64decode(image_data.b64_json))
```
### Multiple Images Edit
```python
from litellm import image_edit
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=[
open("scene.png", "rb"),
open("style_reference.png", "rb"),
],
prompt="Blend the reference style into the scene",
)
print(response)
```

90
docs/my-website/docs/providers/vertex.md
View file

@ -1472,6 +1472,82 @@ Your WIF credentials JSON file typically looks like this (for AWS federation):
For more details on setting up Workload Identity Federation, see [Google Cloud WIF documentation](https://cloud.google.com/iam/docs/workload-identity-federation).
#### Explicit AWS Credentials for WIF
By default, AWS-based WIF relies on the EC2 instance metadata service to obtain AWS credentials. This works when LiteLLM runs on an EC2 instance or ECS task with an IAM role attached.
If your environment **does not have access to the EC2 metadata service** (e.g., running on-premises, in a container without host networking, or in a different cloud with security restrictions), you can provide explicit AWS credentials directly in the WIF credential JSON file. LiteLLM will use these to authenticate to AWS before performing the GCP token exchange.
Add the `aws_*` keys at the **top level** of your WIF credential JSON (alongside `type`, `audience`, etc.):
```json
{
"type": "external_account",
"audience": "//iam.googleapis.com/projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID",
"subject_token_type": "urn:ietf:params:aws:token-type:aws4_request",
"service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/SERVICE_ACCOUNT_EMAIL:generateAccessToken",
"token_url": "https://sts.googleapis.com/v1/token",
"credential_source": {
"environment_id": "aws1",
"region_url": "http://169.254.169.254/latest/meta-data/placement/availability-zone",
"url": "http://169.254.169.254/latest/meta-data/iam/security-credentials",
"regional_cred_verification_url": "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15"
},
"aws_role_name": "arn:aws:iam::123456789012:role/MyWifRole",
"aws_region_name": "us-east-1"
}
```
**Supported `aws_*` parameters:**
| Parameter | Required | Description |
|---|---|---|
| `aws_region_name` | Yes | AWS region for credential verification (e.g. `us-east-1`) |
| `aws_role_name` | No | IAM role ARN for STS AssumeRole |
| `aws_access_key_id` | No | Static AWS access key ID |
| `aws_secret_access_key` | No | Static AWS secret access key |
| `aws_session_token` | No | Temporary session token |
| `aws_profile_name` | No | AWS CLI profile name |
| `aws_session_name` | No | Session name for AssumeRole |
| `aws_web_identity_token` | No | Web identity token for STS |
| `aws_sts_endpoint` | No | Custom STS endpoint URL |
| `aws_external_id` | No | External ID for cross-account AssumeRole |
`aws_region_name` is always required when using explicit AWS credentials. The other parameters follow the same authentication flows as [Bedrock AWS auth](/docs/providers/bedrock#authentication) -- you can use role assumption, static keys, profiles, or web identity tokens.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="vertex_ai/gemini-1.5-pro",
messages=[{"role": "user", "content": "Hello!"}],
vertex_credentials="/path/to/wif-credentials-with-aws.json", # WIF JSON with aws_* keys
vertex_project="your-gcp-project-id",
vertex_location="us-central1"
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: gemini-model
litellm_params:
model: vertex_ai/gemini-1.5-pro
vertex_project: your-gcp-project-id
vertex_location: us-central1
vertex_credentials: /path/to/wif-credentials-with-aws.json # WIF JSON with aws_* keys
```
</TabItem>
</Tabs>
When `aws_*` keys are present in the JSON, LiteLLM automatically uses explicit AWS authentication instead of the EC2 metadata service. When they are absent, the standard metadata-based flow is used unchanged.
### **Environment Variables**
You can set:
@ -1687,6 +1763,20 @@ litellm.vertex_location = "us-central1 # Your Location
| gemini-2.5-flash-lite-preview-09-2025 | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` |
| gemini-3.1-flash-lite-preview | `completion('gemini-3.1-flash-lite-preview', messages)`, `completion('vertex_ai/gemini-3.1-flash-lite-preview', messages)` |
## PayGo / Priority Cost Tracking
LiteLLM automatically tracks spend for Vertex AI Gemini models using the correct pricing tier based on the response's `usageMetadata.trafficType`:
| Vertex AI `trafficType` | LiteLLM `service_tier` | Pricing applied |
|-------------------------|-------------------------|-----------------|
| `ON_DEMAND_PRIORITY` | `priority` | PayGo / priority pricing (`input_cost_per_token_priority`, `output_cost_per_token_priority`) |
| `ON_DEMAND` | standard | Default on-demand pricing |
| `FLEX` / `BATCH` | `flex` | Batch/flex pricing |
When you use [Vertex AI PayGo](https://cloud.google.com/vertex-ai/generative-ai/pricing) (on-demand priority) or batch workloads, LiteLLM reads `trafficType` from the response and applies the matching cost per token from the [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). No configuration is required — spend tracking works out of the box for both standard and PayGo requests.
See [Spend Tracking](../proxy/cost_tracking.md) for general cost tracking setup.
## Private Service Connect (PSC) Endpoints
LiteLLM supports Vertex AI models deployed to Private Service Connect (PSC) endpoints, allowing you to use custom `api_base` URLs for private deployments.

72
docs/my-website/docs/proxy/admin_ui_sso.md
View file

@ -41,12 +41,38 @@ After creating the app, copy your **Client ID** and **Client Secret** from the a
Ensure users are assigned to the app in the **Assignments** tab. If Federation Broker Mode is enabled, you may need to disable it to assign users manually.
#### Step 3: Configure Authorization Server Access Policy
#### Step 3: Set Environment Variables
:::warning Important
This step is required. Without an Access Policy for your app, users will get a `no_matching_policy` error when attempting to log in.
Set the following environment variables. The only difference between the two Okta authorization servers is the endpoint URLs:
**Org Authorization Server** (available on all Okta plans, no additional SKU required):
```bash
GENERIC_CLIENT_ID="<your-client-id>"
GENERIC_CLIENT_SECRET="<your-client-secret>"
GENERIC_AUTHORIZATION_ENDPOINT="https://<your-okta-domain>/oauth2/v1/authorize"
GENERIC_TOKEN_ENDPOINT="https://<your-okta-domain>/oauth2/v1/token"
GENERIC_USERINFO_ENDPOINT="https://<your-okta-domain>/oauth2/v1/userinfo"
PROXY_BASE_URL="https://<your-proxy-base-url>"
```
**Custom Authorization Server** (requires the Okta API Access Management SKU):
```bash
GENERIC_CLIENT_ID="<your-client-id>"
GENERIC_CLIENT_SECRET="<your-client-secret>"
GENERIC_AUTHORIZATION_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/authorize"
GENERIC_TOKEN_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/token"
GENERIC_USERINFO_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/userinfo"
PROXY_BASE_URL="https://<your-proxy-base-url>"
```
:::tip
You can find all OAuth endpoints at `https://<your-okta-domain>/.well-known/openid-configuration`
:::
#### Step 3a: Configure Access Policy (Custom Authorization Server only)
If you are using the Custom Authorization Server, you must configure an Access Policy. Without it, users will get a `no_matching_policy` error. Skip this step if you are using the Org Authorization Server.
1. Go to **Security** → **API**
<Image img={require('../../img/okta_security_api.png')} />
@ -62,21 +88,21 @@ This step is required. Without an Access Policy for your app, users will get a `
See [Okta's Access Policy documentation](https://help.okta.com/en-us/content/topics/security/api-access-management/access-policies.htm) for more details.
#### Step 4: Configure LiteLLM Environment Variables
#### Step 4: Configure Okta Security Settings
**GENERIC_CLIENT_STATE** is recommended for Okta to prevent CSRF attacks:
```bash
GENERIC_CLIENT_ID="<your-client-id>"
GENERIC_CLIENT_SECRET="<your-client-secret>"
GENERIC_AUTHORIZATION_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/authorize"
GENERIC_TOKEN_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/token"
GENERIC_USERINFO_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/userinfo"
GENERIC_CLIENT_STATE="random-string"
PROXY_BASE_URL="https://<your-proxy-base-url>"
```
:::tip
You can find all OAuth endpoints at `https://<your-okta-domain>/.well-known/openid-configuration`
:::
**PKCE (Proof Key for Code Exchange)** — If your Okta application is configured to require PKCE, enable it by setting:
```bash
GENERIC_CLIENT_USE_PKCE="true"
```
LiteLLM will automatically handle PKCE parameter generation and verification during the OAuth flow.
#### Step 5: Test the SSO Flow
@ -91,7 +117,7 @@ You can find all OAuth endpoints at `https://<your-okta-domain>/.well-known/open
|-------|-------|----------|
| `redirect_uri` error | Redirect URI not configured | Add `<proxy_base_url>/sso/callback` to Sign-in redirect URIs in Okta |
| `access_denied` | User not assigned to app | Assign the user in the Assignments tab |
| `no_matching_policy` | Missing Access Policy | Create an Access Policy in the Authorization Server (see Step 3) |
| `no_matching_policy` | Missing Access Policy (Custom Authorization Server only) | Create an Access Policy in the Authorization Server (see Step 3a) |
</TabItem>
<TabItem value="google" label="Google SSO">
@ -456,23 +482,9 @@ PROXY_BASE_URL=http://litellm.platform.com
PROXY_BASE_URL=litellm.platform.com
```
**2. For Okta specifically, ensure GENERIC_CLIENT_STATE is set**
**2. For Okta specifically, ensure `GENERIC_CLIENT_STATE` is set and PKCE is configured if required**
Okta requires the `GENERIC_CLIENT_STATE` parameter:
```bash
GENERIC_CLIENT_STATE="random-string" # Required for Okta
```
### Okta PKCE
If your Okta application is configured to require PKCE (Proof Key for Code Exchange), enable it by setting:
```bash
GENERIC_CLIENT_USE_PKCE="true"
```
This is required when your Okta app settings enforce PKCE for enhanced security. LiteLLM will automatically handle PKCE parameter generation and verification during the OAuth flow.
See [Okta SSO — Step 4: Configure Okta Security Settings](#step-4-configure-okta-security-settings) for details on `GENERIC_CLIENT_STATE` and PKCE configuration.
### Common Configuration Issues

7
docs/my-website/docs/proxy/config_settings.md
View file

@ -199,6 +199,7 @@ router_settings:
| use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. |
| disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
| enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. |
| disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
### general_settings - Reference
@ -354,7 +355,7 @@ router_settings:
| set_verbose | boolean | [DEPRECATED PARAM - see debug docs](./debugging) If true, sets the logging level to verbose. |
| retry_after | int | Time to wait before retrying a request in seconds. Defaults to 0. If `x-retry-after` is received from LLM API, this value is overridden. |
| provider_budget_config | ProviderBudgetConfig | Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. [Further Docs](./provider_budget_routing.md) |
| enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) |
| enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. **Required** for `model_info.max_input_tokens` enforcement. Default: false. [More information here](reliability) |
| model_group_retry_policy | Dict[str, RetryPolicy] | [SDK-only arg] Set retry policy for model groups. |
| context_window_fallbacks | List[Dict[str, List[str]]] | Fallback models for context window violations. |
| redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
@ -803,6 +804,7 @@ router_settings:
| PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
| PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used.
| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MAX_BUDGET_PER_SESSION_TTL | TTL in seconds for session budget counters used by the max-budget-per-session limiter. Default is 3600 (1 hour)
| LITELLM_MAX_ITERATIONS_TTL | TTL in seconds for session iteration counters used by the max-iterations limiter. Default is 3600 (1 hour)
| LITELLM_MAX_STREAMING_DURATION_SECONDS | Maximum duration in seconds allowed for a streaming response. Streams exceeding this duration are terminated with a Timeout error. Default is None (no limit)
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
@ -815,6 +817,7 @@ router_settings:
| LITELLM_TOKEN | Access token for LiteLLM integration
| LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES | When set to "true", routes OpenAI /v1/messages requests through chat/completions instead of the Responses API for Anthropic models. Can also be set via `litellm_settings.use_chat_completions_url_for_anthropic_messages`
| LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution
| LITELLM_WORKER_STARTUP_HOOKS | Comma-separated list of `module.path:function_name` callables to run in each worker process during startup. Runs early in the worker lifecycle (before config/DB loading). Useful for re-initializing per-process state like [gflags](https://github.com/google/python-gflags). See [Worker Startup Hooks](/proxy/worker_startup_hooks) for details
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LITELLM_ASYNCIO_QUEUE_MAXSIZE | Maximum size for asyncio queues (e.g. log queues, spend update queues, and cookbook examples such as realtime audio in `nova_sonic_realtime.py`). Bounds in-memory growth to prevent OOM. Default is 1000.
@ -918,6 +921,7 @@ router_settings:
| PRISMA_HEALTH_WATCHDOG_INTERVAL_SECONDS | Interval in seconds for Prisma health watchdog probes. Default is 30
| PRISMA_HEALTH_WATCHDOG_PROBE_TIMEOUT_SECONDS | Timeout in seconds for each Prisma health probe. Default is 5.0
| PRISMA_RECONNECT_COOLDOWN_SECONDS | Cooldown in seconds between Prisma reconnection attempts. Default is 15
| PRISMA_RECONNECT_ESCALATION_THRESHOLD | Number of consecutive reconnect failures before escalating the reconnection strategy. Default is 3
| PRISMA_WATCHDOG_RECONNECT_TIMEOUT_SECONDS | Timeout in seconds for Prisma watchdog-initiated reconnection. Default is 30.0
| PREDIBASE_API_BASE | Base URL for Predibase API
| PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
@ -940,6 +944,7 @@ router_settings:
| QDRANT_URL | Connection URL for Qdrant database
| QDRANT_VECTOR_SIZE | Vector size for Qdrant operations. Default is 1536
| REDIS_CONNECTION_POOL_TIMEOUT | Timeout in seconds for Redis connection pool. Default is 5
| REDIS_CLUSTER_NODES | JSON-formatted list of Redis cluster startup nodes for Redis Cluster mode. Example: '[{"host": "node1", "port": 6379}]'
| REDIS_HOST | Hostname for Redis server
| REDIS_PASSWORD | Password for Redis service
| REDIS_PORT | Port number for Redis server

2
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -8,6 +8,8 @@ Track spend for keys, users, and teams across 100+ LLMs.
LiteLLM automatically tracks spend for all known models. See our [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
Provider-specific cost tracking (e.g., [Vertex AI PayGo / priority pricing](../providers/vertex.md#paygo--priority-cost-tracking), [Bedrock service tiers](../providers/bedrock.md#usage---service-tier), [Azure base model mapping](./custom_pricing.md#set-base_model-for-cost-tracking-eg-azure-deployments)) is applied automatically when the response includes tier metadata.
:::tip Keep Pricing Data Updated
[Sync model pricing data from GitHub](./sync_models_github.md) to ensure accurate cost tracking.
:::

9
docs/my-website/docs/proxy/custom_pricing.md
View file

@ -104,9 +104,18 @@ There are other keys you can use to specify costs for different scenarios and mo
- `input_cost_per_video_per_second` - Cost per second of video input
- `input_cost_per_video_per_second_above_128k_tokens` - Video cost for large contexts
- `input_cost_per_character` - Character-based pricing for some providers
- `input_cost_per_token_priority` / `output_cost_per_token_priority` - Priority/PayGo pricing (Vertex AI Gemini, Bedrock)
- `input_cost_per_token_flex` / `output_cost_per_token_flex` - Batch/flex pricing
These keys evolve based on how new models handle multimodality. The latest version can be found at [https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
### Service Tier / PayGo Pricing (Vertex AI, Bedrock)
For providers that support multiple pricing tiers (e.g., Vertex AI PayGo, Bedrock service tiers), LiteLLM automatically applies the correct cost based on the response:
- **Vertex AI Gemini**: Uses `usageMetadata.trafficType` (`ON_DEMAND_PRIORITY` → priority, `FLEX`/`BATCH` → flex). See [Vertex AI - PayGo / Priority Cost Tracking](../providers/vertex.md#paygo--priority-cost-tracking).
- **Bedrock**: Uses `serviceTier` from the response. See [Bedrock - Usage - Service Tier](../providers/bedrock.md#usage---service-tier).
## Zero-Cost Models (Bypass Budget Checks)
**Use Case**: You have on-premises or free models that should be accessible even when users exceed their budget limits.

16
docs/my-website/docs/proxy/custom_sso.md
View file

@ -121,15 +121,14 @@ Use this if you want to run your own code **after** a user signs on to the LiteL
Make sure the response type follows the `SSOUserDefinedValues` pydantic object. This is used for logging the user into the Admin UI:
```python
from fastapi import Request
from fastapi_sso.sso.base import OpenID
from litellm.proxy._types import LitellmUserRoles, SSOUserDefinedValues
from litellm.proxy.management_endpoints.internal_user_endpoints import (
new_user,
user_info,
)
from litellm.proxy.management_endpoints.team_endpoints import add_new_member
from litellm.proxy import proxy_server
# These imports are available if you need to create users or manage team membership:
# from litellm.proxy.management_endpoints.internal_user_endpoints import new_user
# from litellm.proxy.management_endpoints.team_endpoints import add_new_member
async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:
@ -158,8 +157,9 @@ async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:
#################################################
# Run your custom code / logic here
# check if user exists in litellm proxy DB
_user_info = await user_info(user_id=userIDPInfo.id)
print("_user_info from litellm DB ", _user_info) # noqa
if proxy_server.prisma_client is not None:
_user_info = await proxy_server.prisma_client.get_data(user_id=userIDPInfo.id)
print("_user_info from litellm DB ", _user_info) # noqa
#################################################
return SSOUserDefinedValues(

2
docs/my-website/docs/proxy/dynamic_rate_limit.md
View file

@ -3,6 +3,8 @@
Prevent projects from gobbling too much tpm/rpm.
**See Also:** [Request Prioritization](../scheduler.md) - Prioritize LLM API requests in high-traffic by adding them to a priority queue.
Dynamically allocate TPM/RPM quota to api keys, based on active keys in that minute. [**See Code**](https://github.com/BerriAI/litellm/blob/9bffa9a48e610cc6886fc2dce5c1815aeae2ad46/litellm/proxy/hooks/dynamic_rate_limiter.py#L125)
## Quick Start Usage

2
docs/my-website/docs/proxy/forward_client_headers.md
View file

@ -112,6 +112,8 @@ general_settings:
forward_llm_provider_auth_headers: true # Enable BYOK
```
For **Claude Code** with `/login` and your own Anthropic key, see [Claude Code BYOK](../tutorials/claude_code_byok.md). Use `ANTHROPIC_CUSTOM_HEADERS="x-litellm-api-key: sk-12345"` to pass your LiteLLM key while your Anthropic key (from `/login`) is forwarded as `x-api-key`.
Client request:
```bash
curl -X POST "http://localhost:4000/v1/messages" \

13
docs/my-website/docs/proxy/guardrails/azure_content_guardrail.md
View file

@ -100,6 +100,19 @@ AzureHarmCategories:
n/a
## Important Notes
### Azure Content Safety Character Limit
Both Azure Prompt Shield and Azure Text Moderation have a **10,000 character limit** per request. When text exceeds this limit:
- LiteLLM automatically splits the text into chunks at word boundaries (no words are broken)
- Each chunk is sent separately to the Azure Content Safety API for analysis
- If any chunk is flagged (attack detected or severity threshold exceeded), the entire request is blocked
- If all chunks are safe, the request is allowed to proceed
This applies to both `pre_call` and `post_call` hooks and ensures that long prompts are properly analyzed without breaking words or losing context.
## Further Reading

43
docs/my-website/docs/proxy/guardrails/quick_start.md
View file

@ -358,13 +358,13 @@ response = client.chat.completions.create(
}
],
extra_body={
"guardrails": [
"guardrails": {
"aporia-pre-guard": {
"extra_body": {
"success_threshold": 0.9
}
}
]
}
}
)
@ -387,13 +387,13 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"content": "what llm are you"
}
],
"guardrails": [
"guardrails": {
"aporia-pre-guard": {
"extra_body": {
"success_threshold": 0.9
}
}
]
}
}'
```
</TabItem>
@ -451,7 +451,6 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Content-Type: application/json' \
-d '{
"guardrails": ["aporia-pre-guard", "aporia-post-guard"]
}
}'
```
@ -465,7 +464,6 @@ curl --location 'http://0.0.0.0:4000/key/update' \
--data '{
"key": "sk-jNm1Zar7XfNdZXp49Z1kSQ",
"guardrails": ["aporia-pre-guard", "aporia-post-guard"]
}
}'
```
@ -499,7 +497,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
Run guardrails based on the user-agent header. This is useful for running pre-call checks on OpenWebUI but only masking in logs for Claude CLI.
`default` can be a single mode string or a list of modes.
Both `default` and tag values can be a single mode string or a list of modes.
<Tabs>
<TabItem value="single" label="Single Default Mode">
@ -547,6 +545,29 @@ guardrails:
default_on: true
```
</TabItem>
<TabItem value="tag-list" label="Multiple Tag Modes">
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "guardrails_ai-guard"
litellm_params:
guardrail: guardrails_ai
guard_name: "pii_detect"
mode:
tags:
"User-Agent: claude-cli": ["pre_call", "post_call"] # Run both pre and post call for claude-cli
default: "logging_only" # Default to logging only when no tags match
api_base: os.environ/GUARDRAILS_AI_API_BASE
default_on: true
```
</TabItem>
</Tabs>
@ -671,7 +692,7 @@ guardrails:
Mode Specification
`default` accepts either a single string or a list of strings.
Both `default` and tag values accept either a single string or a list of strings.
```python
from litellm.types.guardrails import Mode
@ -687,6 +708,12 @@ mode = Mode(
tags={"User-Agent: claude-cli": "logging_only"},
default=["pre_call", "post_call"]
)
# Multiple modes on a tag value
mode = Mode(
tags={"User-Agent: claude-cli": ["pre_call", "post_call"]},
default="logging_only"
)
```
### `guardrails` Request Parameter

28
docs/my-website/docs/proxy/reliability.md
View file

@ -713,6 +713,34 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
[**See Code**](https://github.com/BerriAI/litellm/blob/c9e6b05cfb20dfb17272218e2555d6b496c47f6f/litellm/router.py#L2163)
:::important
**`enable_pre_call_checks` is required** for context-window enforcement. Without it, requests are sent to the provider regardless of input token count. Set `enable_pre_call_checks: true` in `router_settings` in your config.
:::
#### Custom max_input_tokens per deployment
You can override the default context limit for a deployment by setting `max_input_tokens` in `model_info`. This is useful for testing, rate-limiting long prompts, or enforcing stricter limits than the provider's default.
**Both** of the following are required:
1. **`router_settings.enable_pre_call_checks: true`** — enables pre-call checks
2. **`model_info.max_input_tokens`** on the deployment — overrides the limit for that model
```yaml
router_settings:
enable_pre_call_checks: true # Required for enforcement
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
model_info:
max_input_tokens: 10 # Override: reject prompts > 10 tokens
```
If a request exceeds the limit, LiteLLM raises `ContextWindowExceededError` with details like `Model=gpt-4o, Max Input Tokens=10, Got=306`.
**1. Setup config**
For azure deployments, set the base model. Pick the base model from [this list](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json), all the azure models start with azure/.

89
docs/my-website/docs/proxy/token_auth.md
View file

@ -1054,6 +1054,95 @@ curl -X GET 'http://0.0.0.0:4000/user/info?user_id=user-123' \
-H 'Authorization: Bearer <PROXY_MASTER_KEY>'
```
## [BETA] JWT-to-Virtual-Key Mapping
Map JWT identities to LiteLLM virtual keys so that JWT-authenticated users get per-user budgets, rate limits, model access controls, and spend tracking.
When a JWT comes in, LiteLLM looks up a configured claim (e.g. `email`, `sub`) in a mapping table. If a mapping exists, the request is treated as if it arrived with the corresponding virtual key — all virtual key features apply.
### Setup
Add `virtual_key_claim_field` to your JWT auth config:
```yaml
general_settings:
enable_jwt_auth: True
litellm_jwtauth:
virtual_key_claim_field: "email" # JWT claim to look up (supports dot notation)
virtual_key_mapping_cache_ttl: 300 # Cache TTL in seconds (default: 300)
```
### Managing Mappings
All endpoints require admin auth (`Authorization: Bearer <master_key>`).
**Create a mapping** — link a JWT claim value to an existing virtual key:
```bash
curl -X POST http://localhost:4000/jwt/key/mapping/new \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"jwt_claim_name": "email",
"jwt_claim_value": "user@example.com",
"key": "sk-virtual-key-from-key-generate"
}'
```
**List mappings** (paginated):
```bash
curl http://localhost:4000/jwt/key/mapping/list?page=1&size=50 \
-H "Authorization: Bearer sk-1234"
```
**Get a specific mapping:**
```bash
curl "http://localhost:4000/jwt/key/mapping/info?id=<mapping-id>" \
-H "Authorization: Bearer sk-1234"
```
**Update a mapping:**
```bash
curl -X POST http://localhost:4000/jwt/key/mapping/update \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"id": "<mapping-id>",
"description": "Updated description",
"is_active": true
}'
```
**Delete a mapping:**
```bash
curl -X POST http://localhost:4000/jwt/key/mapping/delete \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{"id": "<mapping-id>"}'
```
### How It Works
1. A request arrives with a JWT bearer token
2. LiteLLM validates the JWT signature
3. Extracts the configured claim (e.g. `email` → `user@example.com`)
4. Looks up the claim value in the `LiteLLM_JWTKeyMapping` table
5. If a mapping exists, the request proceeds as if the mapped virtual key was used — budgets, rate limits, model access, and spend tracking all apply
6. If no mapping exists, falls back to standard JWT auth (team-level controls)
### Error Codes
| Code | Meaning |
|------|---------|
| 409 | Duplicate mapping — a mapping for that claim name + value already exists |
| 400 | The provided key does not match an existing virtual key |
| 404 | Mapping not found (for update/delete/info) |
| 403 | Non-admin user attempted a mapping operation |
## All JWT Params
[**See Code**](https://github.com/BerriAI/litellm/blob/b204f0c01c703317d812a1553363ab0cb989d5b6/litellm/proxy/_types.py#L95)

130
docs/my-website/docs/proxy/users.md
View file

@ -10,6 +10,8 @@ import TabItem from '@theme/TabItem';
**Team member budgets**: Set individual spending limits within the team's shared budget
**Agent budgets**: Set rate limits (tpm/rpm) and session-level caps (iterations, dollar budget) on agents [**Jump**](#agents)
***If a key belongs to a team, the team budget is applied, not the user's personal budget.***
:::
@ -420,6 +422,109 @@ Expected response on failure
</Tabs>
### Agents
Set budgets and rate limits on agents registered with LiteLLM's [Agent Gateway](../a2a.md). You can control:
- **Per-agent rate limits**: `tpm_limit` and `rpm_limit` on the agent itself
- **Per-session rate limits**: `session_tpm_limit` and `session_rpm_limit` applied per session
- **Per-session iteration cap**: `max_iterations` in agent `litellm_params`
- **Per-session budget cap**: `max_budget_per_session` in agent `litellm_params`
<Tabs>
<TabItem value="agent-rate-limits" label="Agent Rate Limits">
Set `tpm_limit` and `rpm_limit` on the agent to cap total throughput across all sessions.
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"tpm_limit": 100000,
"rpm_limit": 100
}'
```
</TabItem>
<TabItem value="session-rate-limits" label="Session Rate Limits">
Set `session_tpm_limit` and `session_rpm_limit` to cap throughput per individual session.
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"session_tpm_limit": 50000,
"session_rpm_limit": 50
}'
```
</TabItem>
<TabItem value="session-budgets" label="Session Budgets">
Set `max_iterations` and `max_budget_per_session` in agent `litellm_params` to cap individual sessions. Requires `require_trace_id_on_calls_by_agent` so LiteLLM can track calls per session.
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"litellm_params": {
"require_trace_id_on_calls_by_agent": true,
"max_iterations": 25,
"max_budget_per_session": 5.00
}
}'
```
When a session exceeds the limit, requests receive a **429 Too Many Requests** response.
See the [Agent Iteration Budgets](../a2a_iteration_budgets) guide for full details.
</TabItem>
</Tabs>
:::info
You can also update rate limits on existing agents using `PATCH /v1/agents/{agent_id}`:
```bash
curl -X PATCH 'http://localhost:4000/v1/agents/<agent_id>' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"tpm_limit": 200000,
"rpm_limit": 200,
"session_tpm_limit": 50000,
"session_rpm_limit": 50
}'
```
:::
### Customers
Use this to budget `user` passed to `/chat/completions`, **without needing to create a key for every user**
@ -685,6 +790,31 @@ These headers indicate:
- 1 request remaining for the GPT-4 model for key=`sk-ulGNRXWtv7M0lFnnsQk0wQ`
- 179 tokens remaining for the GPT-4 model for key=`sk-ulGNRXWtv7M0lFnnsQk0wQ`
</TabItem>
<TabItem value="per-agent" label="Per Agent">
Set rate limits on agents registered with the [Agent Gateway](../a2a.md).
**Agent-level limits** cap total throughput across all sessions:
```shell
curl -X POST 'http://0.0.0.0:4000/v1/agents' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{"agent_name": "my-agent", "agent_card_params": {"name": "my-agent", "description": "My agent", "url": "http://my-agent:8080", "version": "1.0.0"}, "tpm_limit": 100000, "rpm_limit": 100}'
```
**Session-level limits** cap throughput per individual session:
```shell
curl -X POST 'http://0.0.0.0:4000/v1/agents' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{"agent_name": "my-agent", "agent_card_params": {"name": "my-agent", "description": "My agent", "url": "http://my-agent:8080", "version": "1.0.0"}, "session_tpm_limit": 50000, "session_rpm_limit": 50}'
```
You can also set **max_iterations** (call count cap) and **max_budget_per_session** (dollar cap) per session via `litellm_params`. See [Agent Iteration Budgets](../a2a_iteration_budgets) for details.
</TabItem>
<TabItem value="per-end-user" label="For customers">

155
docs/my-website/docs/proxy/worker_startup_hooks.md Normal file
View file

@ -0,0 +1,155 @@
# Worker Startup Hooks
Use `LITELLM_WORKER_STARTUP_HOOKS` to run custom initialization functions in **each worker process** during proxy startup. This is essential when using multi-worker deployments (`--num_workers > 1`) with libraries that require per-process initialization, such as [gflags](https://github.com/google/python-gflags).
## The Problem
When running the LiteLLM proxy with multiple workers:
```bash
litellm --config config.yaml --num_workers 4
```
Each worker is a **separate process** spawned by uvicorn or gunicorn. Any in-process state initialized in the master process (before `run_server()`) is **not available** in worker processes. This includes:
- [python-gflags](https://github.com/google/python-gflags) (`gflags.FLAGS`)
- [absl-py flags](https://abseil.io/docs/python/guides/flags) (`absl.flags.FLAGS`)
- Custom singleton registries or connection pools
- Any module-level state that requires explicit initialization
## Usage
Set the `LITELLM_WORKER_STARTUP_HOOKS` environment variable to a comma-separated list of `module.path:function_name` callables:
```bash
export LITELLM_WORKER_STARTUP_HOOKS="my_module:my_init_function"
```
Each hook is called **early** in the worker startup lifecycle — before config loading, database setup, or any request handling. Both sync and async functions are supported.
## Example: gflags Initialization
### 1. Define your wrapper module
```python title="my_litellm_wrapper.py"
import gflags
import json
import os
import sys
from typing import Optional, List, Any
def init_gflags(
usage: Optional[Any] = None,
raw_args: Optional[List[str]] = None,
known_only: bool = False,
) -> List[str]:
"""Initialize gflags from command-line arguments."""
try:
gflags.FLAGS.set_gnu_getopt(True)
if raw_args is None:
raw_args = sys.argv
argv = gflags.FLAGS(raw_args, known_only=known_only)
except gflags.Error as e:
if usage is None:
print("%s\nUsage: %s ARGS\n%s" % (e, sys.argv[0], gflags.FLAGS))
else:
print(usage % dict(cmd=sys.argv[0], flags=gflags.FLAGS))
sys.exit(1)
return argv
def init_gflags_for_worker():
"""Re-initialize gflags in each worker process.
Reads the original sys.argv from the GFLAGS_ARGV env var
(set by the master process before starting the proxy).
"""
raw_args = json.loads(os.environ.get("GFLAGS_ARGV", "[]")) or sys.argv
init_gflags(raw_args=raw_args, known_only=True)
```
### 2. Start the proxy
```python title="start_proxy.py"
import json
import os
import sys
from my_litellm_wrapper import init_gflags
# Store sys.argv so workers can re-parse the same flags
os.environ["GFLAGS_ARGV"] = json.dumps(sys.argv)
# Tell LiteLLM to call our hook in each worker
os.environ["LITELLM_WORKER_STARTUP_HOOKS"] = "my_litellm_wrapper:init_gflags_for_worker"
# Initialize gflags in the master process
init_gflags()
# Start the proxy (programmatic invocation)
from litellm.proxy.proxy_cli import run_server
run_server(
["--config", "config.yaml", "--num_workers", "4"],
standalone_mode=False,
)
```
Or via shell:
```bash
export GFLAGS_ARGV='["my_app", "--my_flag=value", "--batch_size=32"]'
export LITELLM_WORKER_STARTUP_HOOKS="my_litellm_wrapper:init_gflags_for_worker"
litellm --config config.yaml --num_workers 4
```
## How It Works
```
Master Process Worker Process (×N)
───────────────── ──────────────────────
1. init_gflags() 3. proxy_startup_event():
2. run_server() → Read LITELLM_WORKER_STARTUP_HOOKS
→ sets env vars → Import & call each hook
→ uvicorn.run(workers=N) (gflags.FLAGS re-initialized ✓)
→ spawns workers ──────────────────► → Continue with config/DB setup
→ Ready to serve requests
```
- Hooks run at the **very beginning** of `proxy_startup_event` (the FastAPI lifespan), before config loading, database connections, or any other initialization.
- Environment variables set in the master process are **inherited** by worker processes (standard Unix fork/spawn behavior).
- If a hook **raises an exception**, the worker fails to start — this is intentional, since missing initialization (e.g., uninitialized gflags) would cause downstream errors.
## Multiple Hooks
Separate multiple hooks with commas:
```bash
export LITELLM_WORKER_STARTUP_HOOKS="my_module:init_gflags,my_module:init_metrics,my_module:init_connections"
```
Hooks are executed **in order**, left to right.
## Async Hooks
Async functions are also supported — they are automatically awaited:
```python
async def init_async_connections():
"""Example async hook for initializing async resources."""
await setup_async_connection_pool()
```
```bash
export LITELLM_WORKER_STARTUP_HOOKS="my_module:init_async_connections"
```
## Reference
| Environment Variable | Description |
|---|---|
| `LITELLM_WORKER_STARTUP_HOOKS` | Comma-separated `module.path:function_name` callables to run in each worker on startup |
The hook format follows the standard Python entry point syntax: `module.path:function_name`, where `module.path` is a dotted Python import path and `function_name` is the name of the callable within that module.

240
docs/my-website/docs/response_api.md
View file

@ -14,6 +14,7 @@ Requests to /chat/completions may be bridged here automatically when the provide
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Streaming | ✅ | |
| WebSocket Mode | ✅ | Lower-latency persistent connections for all providers |
| Image Generation Streaming | ✅ | Progressive image generation with partial images (1-3) |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
@ -810,6 +811,245 @@ for event in response:
</TabItem>
</Tabs>
## WebSocket Mode
The Responses API supports **WebSocket mode** for lower-latency, persistent connections ideal for agentic workflows. WebSocket mode works with **all LiteLLM providers**, not just those with native WebSocket support.
### Architecture
LiteLLM provides two WebSocket modes:
1. **Native WebSocket**: Direct `wss://` connection to providers that support it (OpenAI, Azure)
2. **Managed WebSocket**: HTTP streaming over WebSocket for all other providers (Anthropic, Gemini, Bedrock, etc.)
The system automatically selects the appropriate mode based on provider capabilities.
### Usage
<Tabs>
<TabItem value="python" label="Python (websocket-client)">
```python showLineNumbers title="WebSocket with Python"
import json
from websocket import create_connection # pip install websocket-client
# Connect to LiteLLM proxy WebSocket endpoint
ws = create_connection(
"ws://localhost:4000/v1/responses?model=gemini-2.5-flash",
header=["Authorization: Bearer sk-1234"]
)
try:
# Send initial message
ws.send(json.dumps({
"type": "response.create",
"model": "gemini-2.5-flash",
"store": True,
"input": [{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "My favorite color is blue."}]
}]
}))
# Collect response events
response_id = None
while True:
event = json.loads(ws.recv())
print(f"Event: {event['type']}")
if event["type"] == "response.completed":
response_id = event["response"]["id"]
break
elif event["type"] == "response.output_text.delta":
print(f"Text: {event.get('delta', '')}", end="", flush=True)
print(f"\nResponse ID: {response_id}")
# Send follow-up with previous_response_id for multi-turn
ws.send(json.dumps({
"type": "response.create",
"model": "gemini-2.5-flash",
"previous_response_id": response_id,
"input": [{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "What is my favorite color?"}]
}]
}))
# Collect follow-up response
while True:
event = json.loads(ws.recv())
if event["type"] == "response.completed":
break
elif event["type"] == "response.output_text.delta":
print(event.get("delta", ""), end="", flush=True)
finally:
ws.close()
```
</TabItem>
<TabItem value="javascript" label="JavaScript (ws)">
```javascript showLineNumbers title="WebSocket with JavaScript"
const WebSocket = require('ws'); // npm install ws
const ws = new WebSocket(
'ws://localhost:4000/v1/responses?model=gemini-2.5-flash',
{
headers: {
'Authorization': 'Bearer sk-1234'
}
}
);
ws.on('open', () => {
// Send initial message
ws.send(JSON.stringify({
type: 'response.create',
model: 'gemini-2.5-flash',
store: true,
input: [{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'My favorite color is blue.' }]
}]
}));
});
let responseId = null;
ws.on('message', (data) => {
const event = JSON.parse(data.toString());
console.log(`Event: ${event.type}`);
if (event.type === 'response.completed') {
responseId = event.response.id;
console.log(`Response ID: ${responseId}`);
// Send follow-up
ws.send(JSON.stringify({
type: 'response.create',
model: 'gemini-2.5-flash',
previous_response_id: responseId,
input: [{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'What is my favorite color?' }]
}]
}));
} else if (event.type === 'response.output_text.delta') {
process.stdout.write(event.delta || '');
}
});
ws.on('error', (error) => {
console.error('WebSocket error:', error);
});
```
</TabItem>
<TabItem value="curl" label="curl (websocat)">
```bash showLineNumbers title="WebSocket with websocat"
# Install websocat: brew install websocat (macOS) or cargo install websocat
# Connect to WebSocket endpoint
websocat "ws://localhost:4000/v1/responses?model=gemini-2.5-flash" \
-H="Authorization: Bearer sk-1234"
# Then send JSON events (paste and press Enter):
{"type":"response.create","model":"gemini-2.5-flash","input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"Hello!"}]}]}
# You'll receive streaming events back:
# {"type":"response.created",...}
# {"type":"response.in_progress",...}
# {"type":"response.output_text.delta","delta":"Hello",...}
# {"type":"response.completed",...}
```
</TabItem>
</Tabs>
### Event Types
WebSocket connections receive Server-Sent Events (SSE) formatted as JSON:
| Event Type | Description |
|------------|-------------|
| `response.created` | Response generation started |
| `response.in_progress` | Response is being generated |
| `response.output_item.added` | New output item (message, tool call, etc.) added |
| `response.output_text.delta` | Incremental text chunk |
| `response.output_text.done` | Text output completed |
| `response.content_part.done` | Content part completed |
| `response.output_item.done` | Output item completed |
| `response.completed` | Full response completed successfully |
| `response.failed` | Response generation failed |
| `response.incomplete` | Response incomplete (e.g., max tokens reached) |
| `error` | Error occurred |
### Multi-Turn Conversations
Use `previous_response_id` to maintain conversation context across multiple WebSocket messages:
```python showLineNumbers title="Multi-turn WebSocket Conversation"
# Turn 1
ws.send(json.dumps({
"type": "response.create",
"model": "gemini-2.5-flash",
"store": True, # Required for multi-turn
"input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]}]
}))
# ... collect events and get response_id from response.completed event ...
# Turn 2 - reference previous response
ws.send(json.dumps({
"type": "response.create",
"model": "gemini-2.5-flash",
"previous_response_id": response_id, # Links to previous turn
"input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Continue"}]}]
}))
```
### Provider Support
| Provider | WebSocket Mode | Notes |
|----------|----------------|-------|
| OpenAI | Native | Direct `wss://` connection to OpenAI |
| Azure OpenAI | Native | Direct `wss://` connection to Azure |
| Anthropic | Managed | HTTP streaming over WebSocket |
| Google AI Studio (Gemini) | Managed | HTTP streaming over WebSocket |
| Vertex AI | Managed | HTTP streaming over WebSocket |
| AWS Bedrock | Managed | HTTP streaming over WebSocket |
| All other providers | Managed | HTTP streaming over WebSocket |
**Note**: Both native and managed modes provide the same event stream format. The difference is transparent to clients.
### Configuration
No special configuration needed. WebSocket mode is automatically available on the `/v1/responses` endpoint when accessed via WebSocket protocol (`ws://` or `wss://`).
For LiteLLM Proxy, ensure your models are configured normally:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gemini-2.5-flash
litellm_params:
model: gemini/gemini-2.5-flash
api_key: os.environ/GEMINI_API_KEY
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
```
Both models will automatically support WebSocket mode at `ws://localhost:4000/v1/responses`.
## Response ID Security
By default, LiteLLM Proxy prevents users from accessing other users' response IDs.

8
docs/my-website/docs/search/index.md
View file

@ -2,7 +2,7 @@
| Feature | Supported |
|---------|-----------|
| Supported Providers | `perplexity`, `tavily`, `parallel_ai`, `exa_ai`, `brave`, `google_pse`, `dataforseo`, `firecrawl`, `searxng`, `linkup` |
| Supported Providers | `perplexity`, `tavily`, `parallel_ai`, `exa_ai`, `brave`, `google_pse`, `dataforseo`, `firecrawl`, `searxng`, `linkup`, `duckduckgo`, `searchapi`, `serper` |
| Cost Tracking | ✅ |
| Logging | ✅ |
| Load Balancing | ❌ |
@ -210,7 +210,7 @@ See the [official Perplexity Search documentation](https://docs.perplexity.ai/ap
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `query` | string or array | Yes | Search query. Can be a single string or array of strings |
| `search_provider` | string | Yes (SDK) | The search provider to use: `"perplexity"`, `"tavily"`, `"parallel_ai"`, `"exa_ai"`, `"brave"`, `"google_pse"`, `"dataforseo"`, `"firecrawl"`, `"searxng"`, or `"linkup"` |
| `search_provider` | string | Yes (SDK) | The search provider to use: `"perplexity"`, `"tavily"`, `"parallel_ai"`, `"exa_ai"`, `"brave"`, `"google_pse"`, `"dataforseo"`, `"firecrawl"`, `"searxng"`, `"linkup"`, `"duckduckgo"`, `"searchapi"`, or `"serper"` |
| `search_tool_name` | string | Yes (Proxy) | Name of the search tool configured in `config.yaml` |
| `max_results` | integer | No | Maximum number of results to return (1-20). Default: 10 |
| `search_domain_filter` | array | No | List of domains to filter results (max 20 domains) |
@ -276,7 +276,9 @@ The response follows Perplexity's search format with the following structure:
| Firecrawl | `FIRECRAWL_API_KEY` | `firecrawl` |
| SearXNG | `SEARXNG_API_BASE` (required) | `searxng` |
| Linkup | `LINKUP_API_KEY` | `linkup` |
| DuckDuckGo | `DUCKDUCKGO_API_BASE` | `duckduckgo` |
| Serper | `SERPER_API_KEY` | `serper` |
| DuckDuckGo | `DUCKDUCKGO_API_BASE` | `duckduckgo` |
| SearchAPI.io | `SEARCHAPI_API_KEY` | `searchapi` |
See the individual provider documentation for detailed setup instructions and provider-specific parameters.

197
docs/my-website/docs/search/searchapi.md Normal file
View file

@ -0,0 +1,197 @@
# SearchAPI.io (Google Search)
Get started by creating a free API key via https://www.searchapi.io/.
SearchAPI.io provides access to Google Search results with a simple API. It supports all Google Search parameters including location, language, time filters, and more.
For complete documentation on all supported parameters, visit https://www.searchapi.io/docs/google.
## LiteLLM Python SDK
```python showLineNumbers title="SearchAPI.io Search"
import os
from litellm import search
os.environ["SEARCHAPI_API_KEY"] = "your-api-key"
response = search(
query="latest AI developments",
search_provider="searchapi",
max_results=10
)
# Access search results
for result in response.results:
print(f"{result.title}: {result.url}")
print(f"Snippet: {result.snippet}\n")
```
### Advanced Usage with SearchAPI.io Parameters
SearchAPI.io supports many Google Search-specific parameters:
```python showLineNumbers title="Advanced SearchAPI.io Parameters"
import os
from litellm import search
os.environ["SEARCHAPI_API_KEY"] = "your-api-key"
response = search(
query="machine learning research",
search_provider="searchapi",
max_results=10,
# Unified parameters
country="US",
search_domain_filter=["arxiv.org", "nature.com"],
# SearchAPI.io specific parameters
gl="us", # Country code
hl="en", # Interface language
time_period="last_month", # Time filter
safe="active", # SafeSearch
device="desktop", # Device type
location="New York" # Geographic location
)
```
## LiteLLM AI Gateway
### 1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
search_tools:
- search_tool_name: google-search
litellm_params:
search_provider: searchapi
api_key: os.environ/SEARCHAPI_API_KEY
```
### 2. Start the proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Test the search endpoint
```bash showLineNumbers title="Test Request"
curl http://0.0.0.0:4000/v1/search/google-search \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"query": "latest AI developments",
"max_results": 10,
"country": "US"
}'
```
## SearchAPI.io Specific Parameters
SearchAPI.io supports many Google Search parameters. Here are some commonly used ones:
| Parameter | Type | Description |
|-----------|------|-------------|
| `gl` | string | Country code (e.g., 'us', 'uk', 'de') |
| `hl` | string | Interface language (e.g., 'en', 'es', 'fr') |
| `location` | string | Geographic location (e.g., 'New York', 'London') |
| `device` | string | Device type: 'desktop', 'mobile', 'tablet' |
| `time_period` | string | Time filter: 'last_hour', 'last_day', 'last_week', 'last_month', 'last_year' |
| `time_period_min` | string | Start date (MM/DD/YYYY) |
| `time_period_max` | string | End date (MM/DD/YYYY) |
| `safe` | string | SafeSearch: 'active' or 'off' |
| `lr` | string | Language restriction (e.g., 'lang_en', 'lang_es') |
| `cr` | string | Country restriction |
| `page` | integer | Page number for pagination |
### Example with Time Filters
```python showLineNumbers title="Search with Time Filter"
response = search(
query="AI breakthroughs",
search_provider="searchapi",
max_results=10,
time_period="last_month"
)
```
### Example with Custom Date Range
```python showLineNumbers title="Search with Custom Date Range"
response = search(
query="AI research papers",
search_provider="searchapi",
max_results=10,
time_period_min="01/01/2024",
time_period_max="03/01/2024"
)
```
### Example with Location
```python showLineNumbers title="Search with Location"
response = search(
query="AI conferences",
search_provider="searchapi",
max_results=10,
location="San Francisco",
gl="us"
)
```
## Response Format
SearchAPI.io returns results in the standard LiteLLM search format:
```json
{
"object": "search",
"results": [
{
"title": "Latest AI Developments",
"url": "https://example.com/ai-news",
"snippet": "Recent breakthroughs in artificial intelligence...",
"date": "2024-01-15"
}
]
}
```
## Rate Limits
SearchAPI.io has different rate limits based on your plan:
- Free tier: 100 requests/month
- Paid plans: Higher limits available
Check your current usage at https://www.searchapi.io/dashboard.
## Error Handling
```python showLineNumbers title="Error Handling"
from litellm import search
import os
os.environ["SEARCHAPI_API_KEY"] = "your-api-key"
try:
response = search(
query="test query",
search_provider="searchapi",
max_results=10
)
print(f"Found {len(response.results)} results")
except Exception as e:
print(f"Search failed: {str(e)}")
```
## Additional Resources
- SearchAPI.io Documentation: https://www.searchapi.io/docs
- API Dashboard: https://www.searchapi.io/dashboard
- Pricing: https://www.searchapi.io/pricing

77
docs/my-website/docs/search/serper.md Normal file
View file

@ -0,0 +1,77 @@
# Serper Search
**Get API Key:** [https://serper.dev](https://serper.dev)
## LiteLLM Python SDK
```python showLineNumbers title="Serper Search"
import os
from litellm import search
os.environ["SERPER_API_KEY"] = "your-api-key"
response = search(
query="latest AI developments",
search_provider="serper",
max_results=5
)
```
## LiteLLM AI Gateway
### 1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-5
litellm_params:
model: gpt-5
api_key: os.environ/OPENAI_API_KEY
search_tools:
- search_tool_name: serper-search
litellm_params:
search_provider: serper
api_key: os.environ/SERPER_API_KEY
```
### 2. Start the proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Test the search endpoint
```bash showLineNumbers title="Test Request"
curl http://0.0.0.0:4000/v1/search/serper-search \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"query": "latest AI developments",
"max_results": 5
}'
```
## Provider-specific Parameters
```python showLineNumbers title="Serper Search with Provider-specific Parameters"
import os
from litellm import search
os.environ["SERPER_API_KEY"] = "your-api-key"
response = search(
query="latest tech news",
search_provider="serper",
max_results=10,
# Serper-specific parameters
gl="us", # Country/geolocation code
hl="en", # Language code
autocorrect=False, # Disable autocorrect
tbs="qdr:d", # Time filter: past day ('qdr:h' hour, 'qdr:w' week, 'qdr:m' month)
page=2 # Page number
)
```

123
docs/my-website/docs/tutorials/claude_code_byok.md Normal file
View file

@ -0,0 +1,123 @@
# Claude Code with Bring Your Own Key (BYOK)
Use Claude Code with your own Anthropic API key through the LiteLLM proxy. When you use Claude's `/login` with your Anthropic account, your API key is sent as `x-api-key`. With BYOK enabled, LiteLLM forwards your key to Anthropic instead of using proxy-configured keys — so you pay Anthropic directly while still benefiting from LiteLLM's routing, logging, and guardrails.
## How It Works
1. **Claude Code `/login`** — You sign in with your Anthropic account; Claude Code sends your Anthropic API key as `x-api-key`.
2. **LiteLLM authentication** — You pass your LiteLLM proxy key via `ANTHROPIC_CUSTOM_HEADERS` so the proxy can authenticate and track your usage.
3. **Key forwarding** — With `forward_llm_provider_auth_headers: true`, LiteLLM forwards your `x-api-key` to Anthropic, giving it precedence over any proxy-configured keys.
## Prerequisites
- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
- Anthropic API key (from [console.anthropic.com](https://console.anthropic.com))
- LiteLLM proxy with a virtual key for authentication
## Step 1: Configure LiteLLM Proxy
Enable forwarding of LLM provider auth headers so your Anthropic key takes precedence:
```yaml title="config.yaml"
model_list:
- model_name: claude-sonnet-4-5
litellm_params:
model: anthropic/claude-sonnet-4-5
# No api_key needed — client's key will be used
litellm_settings:
forward_llm_provider_auth_headers: true # Required for BYOK
```
:::info Why `forward_llm_provider_auth_headers`?
By default, LiteLLM strips `x-api-key` from client requests for security. Setting this to `true` allows client-provided provider keys (like your Anthropic key from `/login`) to be forwarded to Anthropic, overriding any proxy-configured keys.
:::
## Step 2: Create a LiteLLM Virtual Key
Create a virtual key in the LiteLLM UI or via API.
```bash
# Example: Create key via API
curl -X POST "http://localhost:4000/key/generate" \
-H "Authorization: Bearer sk-your-master-key" \
-H "Content-Type: application/json" \
-d '{"key_alias": "claude-code-byok", "models": ["claude-sonnet-4-5"]}'
```
## Step 3: Configure Claude Code
Set environment variables so Claude Code uses LiteLLM and sends your LiteLLM key for proxy auth:
```bash
# Point Claude Code to your LiteLLM proxy
export ANTHROPIC_BASE_URL="http://localhost:4000"
# Model name from your config
export ANTHROPIC_MODEL="claude-sonnet-4-5"
# LiteLLM proxy auth — this is added to every request
# Use x-litellm-api-key so the proxy authenticates you; your Anthropic key goes via x-api-key from /login
export ANTHROPIC_CUSTOM_HEADERS="x-litellm-api-key: sk-12345"
```
Replace `sk-12345` with your actual LiteLLM virtual key.
:::tip Multiple headers
For multiple headers, use newline-separated values:
```bash
export ANTHROPIC_CUSTOM_HEADERS="x-litellm-api-key: sk-12345
x-litellm-user-id: my-user-id"
```
:::
## Step 4: Sign In with Claude Code
1. Launch Claude Code:
```bash
claude
```
2. Use **`/login`** and sign in with your Anthropic account (or use your API key directly).
3. Claude Code will send:
- `x-api-key`: Your Anthropic API key (from `/login`)
- `x-litellm-api-key`: Your LiteLLM key (from `ANTHROPIC_CUSTOM_HEADERS`)
4. LiteLLM authenticates you via `x-litellm-api-key`, then forwards `x-api-key` to Anthropic. Your Anthropic key takes precedence over any proxy-configured key.
## Summary
| Header | Source | Purpose |
|--------|--------|---------|
| `x-api-key` | Claude Code `/login` (Anthropic key) | Sent to Anthropic for API calls |
| `x-litellm-api-key` | `ANTHROPIC_CUSTOM_HEADERS` | Proxy authentication, tracking, rate limits |
## Troubleshooting
### Requests fail with "invalid x-api-key"
- Ensure `forward_llm_provider_auth_headers: true` is set in `litellm_settings` (or `general_settings`).
- Restart the LiteLLM proxy after changing the config.
- Verify you completed `/login` in Claude Code so your Anthropic key is being sent.
### Proxy returns 401
- Check that `ANTHROPIC_CUSTOM_HEADERS` includes `x-litellm-api-key: <your-key>`.
- Ensure the LiteLLM key is valid and has access to the model.
### Proxy key is used instead of my Anthropic key
- Confirm `forward_llm_provider_auth_headers: true` is in your config.
- The setting can be in `litellm_settings` or `general_settings` depending on your config structure.
- Enable debug logging: `LITELLM_LOG=DEBUG` to see which key is being forwarded.
## Related
- [Forward Client Headers](./../proxy/forward_client_headers.md) — Full BYOK and header forwarding docs
- [Claude Code Max Subscription](./claude_code_max_subscription.md) — Using Claude Code with OAuth/Max subscription through LiteLLM

BIN
docs/my-website/img/claude_code_byok_screenshot.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 99 KiB

BIN
docs/my-website/img/mcp_openapi_custom_name_badge.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 144 KiB

BIN
docs/my-website/img/mcp_openapi_tool_edit_panel.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 151 KiB

BIN
docs/my-website/img/mcp_openapi_tools_loaded.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 103 KiB

5358
docs/my-website/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

12
docs/my-website/package.json
View file

@ -15,10 +15,10 @@
},
"dependencies": {
"@docusaurus/core": "3.8.1",
"@docusaurus/plugin-google-gtag": "^3.5.2",
"@docusaurus/plugin-google-gtag": "3.8.1",
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "^3.5.2",
"@docusaurus/theme-mermaid": "^3.5.2",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
@ -61,7 +61,7 @@
"mermaid": ">=11.10.0",
"gray-matter": "4.0.3",
"glob": ">=11.1.0",
"tar": ">=7.5.8",
"tar": ">=7.5.10",
"minimatch": ">=10.2.4",
"diff": ">=8.0.3",
"@isaacs/brace-expansion": ">=5.0.1",
@ -93,6 +93,8 @@
"axios": ">=0.30.2",
"webpack": ">=5.94.0",
"serve-static": ">=1.16.0",
"path-to-regexp": ">=0.1.12"
"path-to-regexp": ">=0.1.12",
"dompurify": ">=3.3.2",
"svgo": ">=3.3.3"
}
}

4
docs/my-website/release_notes/v1.81.14.md
View file

@ -1,5 +1,5 @@
---
title: "[Preview] v1.81.14 - New Gateway Level Guardrails & Compliance Playground"
title: "v1.81.14 - New Gateway Level Guardrails & Compliance Playground"
slug: "v1-81-14"
date: 2026-02-21T00:00:00
authors:
@ -27,7 +27,7 @@ import Image from '@theme/IdealImage';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.81.14.rc.1
ghcr.io/berriai/litellm:main-v1.81.14-stable
```
</TabItem>

7
docs/my-website/release_notes/v1.82.0.md
View file

@ -1,5 +1,5 @@
---
title: "v1.82.0 - Realtime Guardrails, Projects Management, and 10+ Performance Optimizations"
title: "[Preview] v1.82.0 - Realtime Guardrails, Projects Management, and 10+ Performance Optimizations"
slug: "v1-82-0"
date: 2026-02-28T00:00:00
authors:
@ -46,6 +46,11 @@ pip install litellm==1.82.0
- **Guardrail ecosystem expansion** — [Noma v2, Lakera v2 post-call, Singapore regulatory policies (PDPA + MAS), employment discrimination blockers, code execution blocker, guardrail policy versioning, and production monitoring](../../docs/proxy/guardrails) - [PR #21400](https://github.com/BerriAI/litellm/pull/21400), [PR #21783](https://github.com/BerriAI/litellm/pull/21783), [PR #21948](https://github.com/BerriAI/litellm/pull/21948)
- **OpenAI Codex 5.3 — day 0** — [Full support for `gpt-5.3-codex` on OpenAI and Azure, plus `gpt-audio-1.5` and `gpt-realtime-1.5` model coverage](../../docs/providers/openai) - [PR #22035](https://github.com/BerriAI/litellm/pull/22035)
- **10+ performance optimizations** — Streaming hot-path fixes, Redis pipeline batching, database task batching, ModelResponse init skip, and router cache improvements — lower latency and CPU on every request
- **`/v1/messages` → `/responses` routing** — `/v1/messages` requests are now routed to the [Responses API](../../docs/response_api) by default for OpenAI/Azure models
:::danger v1/messages routing change
This version starts routing `/v1/messages` requests to the `/responses` API by default. To opt out and continue using chat/completions, set `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true` or `litellm_settings.use_chat_completions_url_for_anthropic_messages: true` in your config.
:::
---

10
docs/my-website/sidebars.js
View file

@ -154,6 +154,7 @@ const sidebars = {
items: [
"tutorials/claude_responses_api",
"tutorials/claude_code_max_subscription",
"tutorials/claude_code_byok",
"tutorials/claude_code_customer_tracking",
"tutorials/claude_code_prompt_cache_routing",
"tutorials/claude_code_websearch",
@ -310,6 +311,7 @@ const sidebars = {
"proxy/master_key_rotations",
"proxy/model_management",
"proxy/prod",
"proxy/worker_startup_hooks",
"proxy/release_cycle",
],
},
@ -538,8 +540,10 @@ const sidebars = {
items: [
"a2a",
"a2a_invoking_agents",
"a2a_agent_headers",
"a2a_cost_tracking",
"a2a_agent_permissions"
"a2a_agent_permissions",
"a2a_iteration_budgets"
],
},
"assistants",
@ -608,6 +612,7 @@ const sidebars = {
items: [
"mcp",
"mcp_usage",
"mcp_openapi",
"mcp_oauth",
"mcp_public_internet",
"mcp_semantic_filter",
@ -623,6 +628,7 @@ const sidebars = {
items: [
"anthropic_unified/index",
"anthropic_unified/structured_output",
"anthropic_unified/messages_to_responses_mapping",
]
},
"anthropic_count_tokens",
@ -678,6 +684,7 @@ const sidebars = {
"search/firecrawl",
"search/searxng",
"search/linkup",
"search/serper",
]
},
"skills",
@ -794,6 +801,7 @@ const sidebars = {
"providers/bedrock_realtime_with_audio",
"providers/aws_polly",
"providers/bedrock_vector_store",
"providers/bedrock_mantle",
]
},
"providers/litellm_proxy",

239
docs/my-website/src/pages/index.md
View file

@ -7,42 +7,41 @@ https://github.com/BerriAI/litellm
## **Call 100+ LLMs using the OpenAI Input/Output Format**
- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
- Translate inputs to provider's endpoints (`/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, and more)
- [Consistent output](https://docs.litellm.ai/docs/supported_endpoints) - same response format regardless of which provider you use
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy)
## How to use LiteLLM
You can use litellm through either:
1. [LiteLLM Proxy Server](#litellm-proxy-server-llm-gateway) - Server (LLM Gateway) to call 100+ LLMs, load balance, cost tracking across projects
2. [LiteLLM python SDK](#basic-usage) - Python Client to call 100+ LLMs, load balance, cost tracking
### **When to use LiteLLM Proxy Server (LLM Gateway)**
You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs:
:::tip
Use LiteLLM Proxy Server if you want a **central service (LLM Gateway) to access multiple LLMs**
Typically used by Gen AI Enablement / ML PLatform Teams
:::
- LiteLLM Proxy gives you a unified interface to access multiple LLMs (100+ LLMs)
- Track LLM Usage and setup guardrails
- Customize Logging, Guardrails, Caching per project
### **When to use LiteLLM Python SDK**
:::tip
Use LiteLLM Python SDK if you want to use LiteLLM in your **python code**
Typically used by developers building llm projects
:::
- LiteLLM SDK gives you a unified interface to access multiple LLMs (100+ LLMs)
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
<table style={{width: '100%', tableLayout: 'fixed'}}>
<thead>
<tr>
<th style={{width: '14%'}}></th>
<th style={{width: '43%'}}><strong><a href="#litellm-proxy-server-llm-gateway">LiteLLM Proxy Server</a></strong></th>
<th style={{width: '43%'}}><strong><a href="#basic-usage">LiteLLM Python SDK</a></strong></th>
</tr>
</thead>
<tbody>
<tr>
<td style={{width: '14%'}}><strong>Use Case</strong></td>
<td style={{width: '43%'}}>Central service (LLM Gateway) to access multiple LLMs</td>
<td style={{width: '43%'}}>Use LiteLLM directly in your Python code</td>
</tr>
<tr>
<td style={{width: '14%'}}><strong>Who Uses It?</strong></td>
<td style={{width: '43%'}}>Gen AI Enablement / ML Platform Teams</td>
<td style={{width: '43%'}}>Developers building LLM projects</td>
</tr>
<tr>
<td style={{width: '14%'}}><strong>Key Features</strong></td>
<td style={{width: '43%'}}>• Centralized API gateway with authentication & authorization<br />• Multi-tenant cost tracking and spend management per project/user<br />• Per-project customization (logging, guardrails, caching)<br />• Virtual keys for secure access control<br />• Admin dashboard UI for monitoring and management</td>
<td style={{width: '43%'}}>• Direct Python library integration in your codebase<br />• Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - <a href="https://docs.litellm.ai/docs/routing">Router</a><br />• Application-level load balancing and cost tracking<br />• Exception handling with OpenAI-compatible errors<br />• Observability callbacks (Lunary, MLflow, Langfuse, etc.)</td>
</tr>
</tbody>
</table>
## **LiteLLM Python SDK**
@ -67,7 +66,7 @@ import os
os.environ["OPENAI_API_KEY"] = "your-api-key"
response = completion(
model="gpt-3.5-turbo",
model="openai/gpt-5",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
@ -83,13 +82,27 @@ import os
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
response = completion(
model="claude-2",
model="anthropic/claude-sonnet-4-5-20250929",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="xai" label="xAI">
```python
from litellm import completion
import os
## set ENV variables
os.environ["XAI_API_KEY"] = "your-api-key"
response = completion(
model="xai/grok-2-latest",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="vertex" label="VertexAI">
```python
@ -97,11 +110,11 @@ from litellm import completion
import os
# auth: run 'gcloud auth application-default'
os.environ["VERTEX_PROJECT"] = "hardy-device-386718"
os.environ["VERTEX_LOCATION"] = "us-central1"
os.environ["VERTEXAI_PROJECT"] = "hardy-device-386718"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
response = completion(
model="chat-bison",
model="vertex_ai/gemini-1.5-pro",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
@ -212,8 +225,61 @@ response = completion(
</TabItem>
<TabItem value="vercel" label="Vercel AI Gateway">
```python
from litellm import completion
import os
## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for instructions on obtaining a key
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
response = completion(
model="vercel_ai_gateway/openai/gpt-5",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
</Tabs>
### Response Format (OpenAI Chat Completions Format)
```json
{
"id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885",
"created": 1734366691,
"model": "gpt-5",
"object": "chat.completion",
"system_fingerprint": null,
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Hello! As an AI language model, I don't have feelings, but I'm operating properly and ready to assist you with any questions or tasks you may have. How can I help you today?",
"role": "assistant",
"tool_calls": null,
"function_call": null
}
}
],
"usage": {
"completion_tokens": 43,
"prompt_tokens": 13,
"total_tokens": 56,
"completion_tokens_details": null,
"prompt_tokens_details": {
"audio_tokens": null,
"cached_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0
}
}
```
### Responses API
Use `litellm.responses()` for advanced models that support reasoning content like GPT-5, o3, etc.
@ -265,11 +331,11 @@ from litellm import responses
import os
# auth: run 'gcloud auth application-default'
os.environ["VERTEX_PROJECT"] = "jr-smith-386718"
os.environ["VERTEX_LOCATION"] = "us-central1"
os.environ["VERTEXAI_PROJECT"] = "jr-smith-386718"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
response = responses(
model="chat-bison",
model="vertex_ai/gemini-1.5-pro",
messages=[{ "content": "What is the capital of France?","role": "user"}]
)
```
@ -314,7 +380,7 @@ import os
os.environ["OPENAI_API_KEY"] = "your-api-key"
response = completion(
model="gpt-3.5-turbo",
model="openai/gpt-5",
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
@ -331,14 +397,29 @@ import os
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
response = completion(
model="claude-2",
model="anthropic/claude-sonnet-4-5-20250929",
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
```
</TabItem>
<TabItem value="xai" label="xAI">
```python
from litellm import completion
import os
## set ENV variables
os.environ["XAI_API_KEY"] = "your-api-key"
response = completion(
model="xai/grok-2-latest",
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
```
</TabItem>
<TabItem value="vertex" label="VertexAI">
```python
@ -346,11 +427,11 @@ from litellm import completion
import os
# auth: run 'gcloud auth application-default'
os.environ["VERTEX_PROJECT"] = "hardy-device-386718"
os.environ["VERTEX_LOCATION"] = "us-central1"
os.environ["VERTEXAI_PROJECT"] = "hardy-device-386718"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
response = completion(
model="chat-bison",
model="vertex_ai/gemini-1.5-pro",
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
@ -370,7 +451,7 @@ os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url"
response = completion(
model="nvidia_nim/<model_name>",
messages=[{ "content": "Hello, how are you?","role": "user"}]
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
```
@ -466,22 +547,74 @@ response = completion(
```
</TabItem>
<TabItem value="vercel" label="Vercel AI Gateway">
```python
from litellm import completion
import os
## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for instructions on obtaining a key
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
response = completion(
model="vercel_ai_gateway/openai/gpt-5",
messages = [{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
```
</TabItem>
</Tabs>
### Streaming Response Format (OpenAI Format)
```json
{
"id": "chatcmpl-2be06597-eb60-4c70-9ec5-8cd2ab1b4697",
"created": 1734366925,
"model": "claude-sonnet-4-5-20250929",
"object": "chat.completion.chunk",
"system_fingerprint": null,
"choices": [
{
"finish_reason": null,
"index": 0,
"delta": {
"content": "Hello",
"role": "assistant",
"function_call": null,
"tool_calls": null,
"audio": null
},
"logprobs": null
}
]
}
```
### Exception handling
LiteLLM maps exceptions across all supported providers to the OpenAI exceptions. All our exceptions inherit from OpenAI's exception types, so any error-handling you have for that, should work out of the box with LiteLLM.
```python
from openai.error import OpenAIError
import litellm
from litellm import completion
import os
os.environ["ANTHROPIC_API_KEY"] = "bad-key"
try:
# some code
completion(model="claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}])
except OpenAIError as e:
print(e)
completion(model="anthropic/claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}])
except litellm.AuthenticationError as e:
# Thrown when the API key is invalid
print(f"Authentication failed: {e}")
except litellm.RateLimitError as e:
# Thrown when you've exceeded your rate limit
print(f"Rate limited: {e}")
except litellm.APIError as e:
# Thrown for general API errors
print(f"API error: {e}")
```
### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
@ -502,7 +635,7 @@ os.environ["OPENAI_API_KEY"]
litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to lunary, mlflow, langfuse, helicone
#openai call
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
response = completion(model="openai/gpt-5", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
```
### Track Costs, Usage, Latency for streaming
@ -527,7 +660,7 @@ litellm.success_callback = [track_cost_callback] # set custom callback function
# litellm.completion() call
response = completion(
model="gpt-3.5-turbo",
model="openai/gpt-5",
messages=[
{
"role": "user",
@ -584,7 +717,7 @@ Example `litellm_config.yaml`
```yaml
model_list:
- model_name: gpt-3.5-turbo
- model_name: gpt-5
litellm_params:
model: azure/<your-azure-model-deployment>
api_base: os.environ/AZURE_API_BASE # runs os.getenv("AZURE_API_BASE")
@ -621,7 +754,7 @@ docker run \
import openai # openai v1.0.0+
client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
response = client.chat.completions.create(model="gpt-5", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"

4
enterprise/litellm_enterprise/integrations/custom_guardrail.py
View file

@ -50,8 +50,10 @@ class EnterpriseCustomGuardrailHelper:
break
if matched_mode is not None:
# Tag matched: only run if event_type matches the tag's mode value
# Tag matched: only run if event_type matches the tag's mode value(s)
if event_type is not None:
if isinstance(matched_mode, list):
return event_type.value in matched_mode
return event_type.value == matched_mode
return True

22
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -78,8 +78,6 @@ class CheckBatchCost:
"status": {"not_in": ["failed", "expired", "cancelled"]}
}
)
completed_jobs = []
for job in jobs:
# get the model from the job
unified_object_id = job.unified_object_id
@ -237,10 +235,16 @@ class CheckBatchCost:
)
# mark the job as complete
completed_jobs.append(job)
if len(completed_jobs) > 0:
await self.prisma_client.db.litellm_managedobjecttable.update_many(
where={"id": {"in": [job.id for job in completed_jobs]}},
data={"batch_processed": True, "status": "complete"},
)
try:
await self.prisma_client.db.litellm_managedobjecttable.update(
where={"id": job.id},
data={
"batch_processed": True,
"status": "complete",
"file_object": response.model_dump_json(),
},
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)

2
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.33"
version = "0.1.34"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"

4
litellm-js/spend-logs/package.json
View file

@ -12,8 +12,8 @@
},
"overrides": {
"glob": ">=11.1.0",
"tar": ">=7.5.8",
"minimatch": ">=10.2.1",
"tar": ">=7.5.10",
"minimatch": ">=10.2.4",
"diff": ">=8.0.3",
"@isaacs/brace-expansion": ">=5.0.1",
"@babel/traverse": ">=7.23.2",

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.51-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.51.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.52-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.52.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.53-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.53.tar.gz vendored Normal file
View file

Binary file not shown.

20
litellm-proxy-extras/litellm_proxy_extras/migrations/20260303000000_update_tool_table_policies/migration.sql Normal file
View file

@ -0,0 +1,20 @@
-- Rename call_policy to input_policy
ALTER TABLE "LiteLLM_ToolTable" RENAME COLUMN "call_policy" TO "input_policy";
-- Add output_policy column
ALTER TABLE "LiteLLM_ToolTable" ADD COLUMN "output_policy" TEXT NOT NULL DEFAULT 'untrusted';
-- Add user_agent column
ALTER TABLE "LiteLLM_ToolTable" ADD COLUMN "user_agent" TEXT;
-- Add last_used_at column
ALTER TABLE "LiteLLM_ToolTable" ADD COLUMN "last_used_at" TIMESTAMP(3);
-- Drop old index on call_policy
DROP INDEX IF EXISTS "LiteLLM_ToolTable_call_policy_idx";
-- CreateIndex
CREATE INDEX "LiteLLM_ToolTable_input_policy_idx" ON "LiteLLM_ToolTable"("input_policy");
-- CreateIndex
CREATE INDEX "LiteLLM_ToolTable_output_policy_idx" ON "LiteLLM_ToolTable"("output_policy");

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260304175016_add_spend_to_agent_table/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260305000000_add_agent_headers/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- Add static_headers and extra_headers to LiteLLM_AgentsTable
ALTER TABLE "LiteLLM_AgentsTable"
ADD COLUMN IF NOT EXISTS "static_headers" JSONB DEFAULT '{}',
ADD COLUMN IF NOT EXISTS "extra_headers" TEXT[] DEFAULT ARRAY[]::TEXT[];

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260305000000_add_rate_limits_to_agents/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable
ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "tpm_limit" INTEGER;
ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "rpm_limit" INTEGER;
ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "session_tpm_limit" INTEGER;
ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "session_rpm_limit" INTEGER;

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260306175056_add_configs_override_table/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "spec_path" TEXT;

57
litellm-proxy-extras/litellm_proxy_extras/migrations/20260306233848_schema_sync/migration.sql Normal file
View file

@ -0,0 +1,57 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "byok_api_key_help_url" TEXT,
ADD COLUMN "byok_description" TEXT[] DEFAULT ARRAY[]::TEXT[],
ADD COLUMN "is_byok" BOOLEAN NOT NULL DEFAULT false,
ADD COLUMN "tool_name_to_description" JSONB DEFAULT '{}',
ADD COLUMN "tool_name_to_display_name" JSONB DEFAULT '{}';
-- CreateTable
CREATE TABLE "LiteLLM_MCPUserCredentials" (
"id" TEXT NOT NULL,
"user_id" TEXT NOT NULL,
"server_id" TEXT NOT NULL,
"credential_b64" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "LiteLLM_MCPUserCredentials_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "LiteLLM_JWTKeyMapping" (
"id" TEXT NOT NULL,
"jwt_claim_name" TEXT NOT NULL,
"jwt_claim_value" TEXT NOT NULL,
"token" TEXT NOT NULL,
"description" TEXT,
"is_active" BOOLEAN NOT NULL DEFAULT true,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT,
CONSTRAINT "LiteLLM_JWTKeyMapping_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "LiteLLM_ConfigOverrides" (
"config_type" TEXT NOT NULL,
"config_value" JSONB NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
CONSTRAINT "LiteLLM_ConfigOverrides_pkey" PRIMARY KEY ("config_type")
);
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_MCPUserCredentials_user_id_server_id_key" ON "LiteLLM_MCPUserCredentials"("user_id", "server_id");
-- CreateIndex
CREATE INDEX "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_is_act_idx" ON "LiteLLM_JWTKeyMapping"("jwt_claim_name", "jwt_claim_value", "is_active");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_key" ON "LiteLLM_JWTKeyMapping"("jwt_claim_name", "jwt_claim_value");
-- AddForeignKey
ALTER TABLE "LiteLLM_JWTKeyMapping" ADD CONSTRAINT "LiteLLM_JWTKeyMapping_token_fkey" FOREIGN KEY ("token") REFERENCES "LiteLLM_VerificationToken"("token") ON DELETE RESTRICT ON UPDATE CASCADE;

86
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -63,9 +63,16 @@ model LiteLLM_AgentsTable {
agent_name String @unique
litellm_params Json?
agent_card_params Json
static_headers Json? @default("{}")
extra_headers String[] @default([])
agent_access_groups String[] @default([])
object_permission_id String?
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
spend Float @default(0.0)
tpm_limit Int?
rpm_limit Int?
session_tpm_limit Int?
session_rpm_limit Int?
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
@ -277,6 +284,7 @@ model LiteLLM_MCPServerTable {
alias String?
description String?
url String?
spec_path String?
transport String @default("sse")
auth_type String?
credentials Json? @default("{}")
@ -287,6 +295,8 @@ model LiteLLM_MCPServerTable {
mcp_info Json? @default("{}")
mcp_access_groups String[]
allowed_tools String[] @default([])
tool_name_to_display_name Json? @default("{}")
tool_name_to_description Json? @default("{}")
extra_headers String[] @default([])
static_headers Json? @default("{}")
// Health check status
@ -302,6 +312,21 @@ model LiteLLM_MCPServerTable {
registration_url String?
allow_all_keys Boolean @default(false)
available_on_public_internet Boolean @default(true)
is_byok Boolean @default(false)
byok_description String[] @default([])
byok_api_key_help_url String?
}
// Per-user BYOK credentials for MCP servers
model LiteLLM_MCPUserCredentials {
id String @id @default(uuid())
user_id String
server_id String
credential_b64 String
created_at DateTime @default(now()) @map("created_at")
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
@@unique([user_id, server_id])
}
// Generate Tokens for Proxy
@ -352,6 +377,7 @@ model LiteLLM_VerificationToken {
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
litellm_project_table LiteLLM_ProjectTable? @relation(fields: [project_id], references: [project_id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
jwt_key_mappings LiteLLM_JWTKeyMapping[]
// SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub"
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."user_id" = $1 OFFSET $2
@ -364,6 +390,24 @@ model LiteLLM_VerificationToken {
@@index([budget_reset_at, expires])
}
model LiteLLM_JWTKeyMapping {
id String @id @default(uuid())
jwt_claim_name String // e.g. "sub", "email"
jwt_claim_value String // The claim value to match
token String // Hashed virtual key (FK)
description String?
is_active Boolean @default(true)
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token])
@@unique([jwt_claim_name, jwt_claim_value])
@@index([jwt_claim_name, jwt_claim_value, is_active])
}
// Deprecated keys during grace period - allows old key to work until revoke_at
model LiteLLM_DeprecatedVerificationToken {
id String @id @default(uuid())
@ -1018,6 +1062,14 @@ model LiteLLM_UISettings {
updated_at DateTime @updatedAt
}
// Generic config overrides table - one row per config_type
model LiteLLM_ConfigOverrides {
config_type String @id
config_value Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}
// Skills table for storing LiteLLM-managed skills
model LiteLLM_SkillsTable {
skill_id String @id @default(uuid())
@ -1076,24 +1128,24 @@ model LiteLLM_PolicyAttachmentTable {
updated_by String?
}
// Global tool registry - auto-discovered from LLM responses; admins set input_policy/output_policy here
// Global tool registry - auto-discovered from LLM responses; admins set input/output policies here
model LiteLLM_ToolTable {
tool_id String @id @default(uuid())
tool_name String @unique // e.g. "huggingface_remote-mcp__dynamic_space"
origin String? // MCP server name or "user_defined"
input_policy String @default("untrusted") // "trusted" | "untrusted" | "blocked"
output_policy String @default("untrusted") // "trusted" | "untrusted"
call_count Int @default(0) // cumulative number of times this tool was seen
assignments Json? @default("{}")
key_hash String? // hash of the virtual key that first called this tool
team_id String? // team that first called this tool
key_alias String? // human-readable alias of the virtual key
user_agent String? // user-agent of the first request that discovered this tool
last_used_at DateTime? // timestamp of the most recent call
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
tool_id String @id @default(uuid())
tool_name String @unique // e.g. "huggingface_remote-mcp__dynamic_space"
origin String? // MCP server name or "user_defined"
input_policy String @default("untrusted") // "trusted" | "untrusted" | "blocked"
output_policy String @default("untrusted") // "trusted" | "untrusted"
call_count Int @default(0) // cumulative number of times this tool was seen
assignments Json? @default("{}")
key_hash String? // hash of the virtual key that first called this tool
team_id String? // team that first called this tool
key_alias String? // human-readable alias of the virtual key
user_agent String? // user-agent of the first request that discovered this tool
last_used_at DateTime? // timestamp of the most recent call
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
@@index([input_policy])
@@index([output_policy])

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.50"
version = "0.4.53"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.50"
version = "0.4.53"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

10
litellm/__init__.py
View file

@ -305,6 +305,9 @@ return_response_headers: bool = (
False # get response headers from LLM Api providers - example x-remaining-requests,
)
enable_json_schema_validation: bool = False
enable_key_alias_format_validation: bool = (
False # opt-in validation of key_alias format on /key/generate and /key/update
)
####################
logging: bool = True
enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
@ -594,6 +597,7 @@ minimax_models: Set = set()
aws_polly_models: Set = set()
gigachat_models: Set = set()
llamagate_models: Set = set()
bedrock_mantle_models: Set = set()
def is_bedrock_pricing_only_model(key: str) -> bool:
@ -858,6 +862,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
gigachat_models.add(key)
elif value.get("litellm_provider") == "llamagate":
llamagate_models.add(key)
elif value.get("litellm_provider") == "bedrock_mantle":
bedrock_mantle_models.add(key)
add_known_models()
@ -966,6 +972,7 @@ model_list = list(
| ovhcloud_models
| lemonade_models
| docker_model_runner_models
| bedrock_mantle_models
| set(clarifai_models)
)
@ -1070,6 +1077,7 @@ models_by_provider: dict = {
"aws_polly": aws_polly_models,
"gigachat": gigachat_models,
"llamagate": llamagate_models,
"bedrock_mantle": bedrock_mantle_models
}
# mapping for those models which have larger equivalents
@ -1251,6 +1259,7 @@ from .ocr.main import *
from .rag.main import *
from .search.main import *
from .realtime_api.main import _arealtime
from .responses.main import _aresponses_websocket
from .fine_tuning.main import *
from .files.main import *
from .vector_store_files.main import (
@ -1430,6 +1439,7 @@ if TYPE_CHECKING:
from .llms.topaz.image_variations.transformation import TopazImageVariationConfig as TopazImageVariationConfig
from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig as OpenAITextCompletionConfig
from .llms.groq.chat.transformation import GroqChatConfig as GroqChatConfig
from .llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig as BedrockMantleChatConfig
from .llms.a2a.chat.transformation import A2AConfig as A2AConfig
from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig as VoyageEmbeddingConfig
from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig

2
litellm/_lazy_imports_registry.py
View file

@ -214,6 +214,7 @@ LLM_CONFIG_NAMES = (
"TopazImageVariationConfig",
"OpenAITextCompletionConfig",
"GroqChatConfig",
"BedrockMantleChatConfig",
"A2AConfig",
"GenAIHubOrchestrationConfig",
"VoyageEmbeddingConfig",
@ -858,6 +859,7 @@ _LLM_CONFIGS_IMPORT_MAP = {
"OpenAITextCompletionConfig",
),
"GroqChatConfig": (".llms.groq.chat.transformation", "GroqChatConfig"),
"BedrockMantleChatConfig": (".llms.bedrock_mantle.chat.transformation", "BedrockMantleChatConfig"),
"A2AConfig": (".llms.a2a.chat.transformation", "A2AConfig"),
"GenAIHubOrchestrationConfig": (
".llms.sap.chat.transformation",

143
litellm/a2a_protocol/main.py
View file

@ -162,6 +162,49 @@ async def _send_message_via_completion_bridge(
return LiteLLMSendMessageResponse.from_dict(response_dict)
async def _execute_a2a_send_with_retry(
a2a_client: Any,
request: Any,
agent_card: Any,
card_url: Optional[str],
api_base: Optional[str],
agent_name: Optional[str],
) -> Any:
"""Send an A2A message with retry logic for localhost URL errors."""
a2a_response = None
for _ in range(2): # max 2 attempts: original + 1 retry
try:
a2a_response = await a2a_client.send_message(request)
break # success, exit retry loop
except A2ALocalhostURLError as e:
a2a_client = handle_a2a_localhost_retry(
error=e,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=False,
)
card_url = agent_card.url if agent_card else None
except Exception as e:
try:
map_a2a_exception(e, card_url, api_base, model=agent_name)
except A2ALocalhostURLError as localhost_err:
a2a_client = handle_a2a_localhost_retry(
error=localhost_err,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=False,
)
card_url = agent_card.url if agent_card else None
continue
except Exception:
raise
if a2a_response is None:
raise RuntimeError(
"A2A send_message failed: no response received after retry attempts."
)
return a2a_response
@client
async def asend_message(
a2a_client: Optional["A2AClientType"] = None,
@ -169,6 +212,7 @@ async def asend_message(
api_base: Optional[str] = None,
litellm_params: Optional[Dict[str, Any]] = None,
agent_id: Optional[str] = None,
agent_extra_headers: Optional[Dict[str, str]] = None,
**kwargs: Any,
) -> LiteLLMSendMessageResponse:
"""
@ -250,9 +294,12 @@ async def asend_message(
"Either a2a_client or api_base is required for standard A2A flow"
)
trace_id = trace_id or str(uuid.uuid4())
extra_headers = {"X-LiteLLM-Trace-Id": trace_id}
extra_headers: Dict[str, str] = {"X-LiteLLM-Trace-Id": trace_id}
if agent_id:
extra_headers["X-LiteLLM-Agent-Id"] = agent_id
# Overlay agent-level headers (agent headers take precedence over LiteLLM internal ones)
if agent_extra_headers:
extra_headers.update(agent_extra_headers)
a2a_client = await create_a2a_client(
base_url=api_base, extra_headers=extra_headers
)
@ -271,47 +318,25 @@ async def asend_message(
card_url = getattr(agent_card, "url", None) if agent_card else None
context_id = trace_id or str(uuid.uuid4())
if request.params.message.context_id is None:
request.params.message.context_id = context_id
message = request.params.message
if isinstance(message, dict):
if message.get("context_id") is None:
message["context_id"] = context_id
else:
if getattr(message, "context_id", None) is None:
message.context_id = context_id
# Retry loop: if connection fails due to localhost URL in agent card, retry with fixed URL
a2a_response = None
for _ in range(2): # max 2 attempts: original + 1 retry
try:
a2a_response = await a2a_client.send_message(request)
break # success, exit retry loop
except A2ALocalhostURLError as e:
# Localhost URL error - fix and retry
a2a_client = handle_a2a_localhost_retry(
error=e,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=False,
)
card_url = agent_card.url if agent_card else None
except Exception as e:
# Map exception - will raise A2ALocalhostURLError if applicable
try:
map_a2a_exception(e, card_url, api_base, model=agent_name)
except A2ALocalhostURLError as localhost_err:
# Localhost URL error - fix and retry
a2a_client = handle_a2a_localhost_retry(
error=localhost_err,
agent_card=agent_card,
a2a_client=a2a_client,
is_streaming=False,
)
card_url = agent_card.url if agent_card else None
continue
except Exception:
# Re-raise the mapped exception
raise
a2a_response = await _execute_a2a_send_with_retry(
a2a_client=a2a_client,
request=request,
agent_card=agent_card,
card_url=card_url,
api_base=api_base,
agent_name=agent_name,
)
verbose_logger.info(f"A2A send_message completed, request_id={request.id}")
# a2a_response is guaranteed to be set if we reach here (loop breaks on success or raises)
assert a2a_response is not None
# Wrap in LiteLLM response type for _hidden_params support
response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response)
@ -413,7 +438,7 @@ def _build_streaming_logging_obj(
return logging_obj
async def asend_message_streaming(
async def asend_message_streaming( # noqa: PLR0915
a2a_client: Optional["A2AClientType"] = None,
request: Optional["SendStreamingMessageRequest"] = None,
api_base: Optional[str] = None,
@ -421,6 +446,7 @@ async def asend_message_streaming(
agent_id: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
proxy_server_request: Optional[Dict[str, Any]] = None,
agent_extra_headers: Optional[Dict[str, str]] = None,
) -> AsyncIterator[Any]:
"""
Async: Send a streaming message to an A2A agent.
@ -502,7 +528,17 @@ async def asend_message_streaming(
raise ValueError(
"Either a2a_client or api_base is required for standard A2A flow"
)
a2a_client = await create_a2a_client(base_url=api_base)
# Mirror the non-streaming path: always include trace and agent-id headers
streaming_extra_headers: Dict[str, str] = {
"X-LiteLLM-Trace-Id": str(request.id),
}
if agent_id:
streaming_extra_headers["X-LiteLLM-Agent-Id"] = agent_id
if agent_extra_headers:
streaming_extra_headers.update(agent_extra_headers)
a2a_client = await create_a2a_client(
base_url=api_base, extra_headers=streaming_extra_headers
)
# Type assertion: a2a_client is guaranteed to be non-None here
assert a2a_client is not None
@ -616,17 +652,30 @@ async def create_a2a_client(
verbose_logger.info(f"Creating A2A client for {base_url}")
# Use LiteLLM's cached httpx client
http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.A2A,
params={"timeout": timeout},
# Use get_async_httpx_client with per-agent params so that different agents
# (with different extra_headers) get separate cached clients. The params
# dict is hashed into the cache key, keeping agent auth isolated while
# still reusing connections within the same agent.
#
# Only pass params that AsyncHTTPHandler.__init__ accepts (e.g. timeout).
# Use "disable_aiohttp_transport" key for cache-key-only data (it's
# filtered out before reaching the constructor).
_client_params: dict = {"timeout": timeout}
if extra_headers:
# Encode headers into a cache-key-only param so each unique header
# set produces a distinct cache key.
_client_params["disable_aiohttp_transport"] = str(
sorted(extra_headers.items())
)
_async_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.A2AProvider,
params=_client_params,
)
httpx_client = http_handler.client
httpx_client = _async_handler.client
if extra_headers:
httpx_client.headers.update(extra_headers)
verbose_proxy_logger.debug(
f"A2A client created with extra_headers={extra_headers}"
f"A2A client created with extra_headers={list(extra_headers.keys())}"
)
# Resolve agent card

13
litellm/batches/batch_utils.py
View file

@ -1,14 +1,10 @@
import json
import time
from typing import Any, List, Literal, Optional, Tuple
import httpx
import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.types.llms.openai import Batch
from litellm.types.utils import CallTypes, ModelInfo, ModelResponse, Usage
from litellm.types.utils import CallTypes, ModelInfo, Usage
from litellm.utils import token_counter
@ -202,9 +198,8 @@ async def _get_batch_output_file_content_as_dictionary(
Required for Azure and other providers that need authentication
"""
from litellm.files.main import afile_content
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
)
from litellm.proxy.openai_files_endpoints.common_utils import \
_is_base64_encoded_unified_file_id
if custom_llm_provider == "vertex_ai":
raise ValueError("Vertex AI does not support file content retrieval")
@ -231,7 +226,7 @@ async def _get_batch_output_file_content_as_dictionary(
credentials = _extract_file_access_credentials(litellm_params)
file_content_kwargs.update(credentials)
_file_content = await afile_content(**file_content_kwargs)
_file_content = await afile_content(**file_content_kwargs) # type: ignore[reportArgumentType]
return _get_file_content_as_dictionary(_file_content.content)

3
litellm/batches/main.py
View file

@ -33,6 +33,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
CancelBatchRequest,
CreateBatchRequest,
FileExpiresAfter,
RetrieveBatchRequest,
)
from litellm.types.router import GenericLiteLLMParams
@ -219,7 +220,7 @@ def create_batch( # noqa: PLR0915
extra_body=extra_body,
)
if output_expires_after is not None:
_create_batch_request["output_expires_after"] = output_expires_after
_create_batch_request["output_expires_after"] = cast(FileExpiresAfter, output_expires_after)
if model is not None:
provider_config = ProviderConfigManager.get_provider_batches_config(
model=model,

8
litellm/caching/caching.py
View file

@ -166,6 +166,14 @@ class Cache:
None. Cache is set as a litellm param
"""
if type == LiteLLMCacheType.REDIS:
# Check REDIS_CLUSTER_NODES env var if no explicit startup nodes
if not redis_startup_nodes:
_env_cluster_nodes = litellm.get_secret("REDIS_CLUSTER_NODES")
if _env_cluster_nodes is not None and isinstance(
_env_cluster_nodes, str
):
redis_startup_nodes = json.loads(_env_cluster_nodes)
if redis_startup_nodes:
# Only pass GCP parameters if they are provided
cluster_kwargs = {

4
litellm/caching/dual_cache.py
View file

@ -346,6 +346,8 @@ class DualCache(BaseCache):
)
try:
if self.in_memory_cache is not None:
if "ttl" not in kwargs and self.default_in_memory_ttl is not None:
kwargs["ttl"] = self.default_in_memory_ttl
await self.in_memory_cache.async_set_cache(key, value, **kwargs)
if self.redis_cache is not None and local_only is False:
@ -367,6 +369,8 @@ class DualCache(BaseCache):
)
try:
if self.in_memory_cache is not None:
if "ttl" not in kwargs and self.default_in_memory_ttl is not None:
kwargs["ttl"] = self.default_in_memory_ttl
await self.in_memory_cache.async_set_cache_pipeline(
cache_list=cache_list, **kwargs
)

33
litellm/caching/llm_caching_handler.py
View file

@ -3,36 +3,21 @@ Add the event loop to the cache key, to prevent event loop closed errors.
"""
import asyncio
from typing import Set
from .in_memory_cache import InMemoryCache
class LLMClientCache(InMemoryCache):
# Background tasks must be stored to prevent garbage collection, which would
# trigger "coroutine was never awaited" warnings. See:
# https://docs.python.org/3/library/asyncio-task.html#creating-tasks
# Intentionally shared across all instances as a global task registry.
_background_tasks: Set[asyncio.Task] = set()
"""Cache for LLM HTTP clients (OpenAI, Azure, httpx, etc.).
def _remove_key(self, key: str) -> None:
"""Close async clients before evicting them to prevent connection pool leaks."""
value = self.cache_dict.get(key)
super()._remove_key(key)
if value is not None:
close_fn = getattr(value, "aclose", None) or getattr(value, "close", None)
if close_fn and asyncio.iscoroutinefunction(close_fn):
try:
task = asyncio.get_running_loop().create_task(close_fn())
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
except RuntimeError:
pass
elif close_fn and callable(close_fn):
try:
close_fn()
except Exception:
pass
IMPORTANT: This cache intentionally does NOT close clients on eviction.
Evicted clients may still be in use by in-flight requests. Closing them
eagerly causes ``RuntimeError: Cannot send a request, as the client has
been closed.`` errors in production after the TTL (1 hour) expires.
Clients that are no longer referenced will be garbage-collected normally.
For explicit shutdown cleanup, use ``close_litellm_async_clients()``.
"""
def update_cache_key_with_event_loop(self, key):
"""

30
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -390,6 +390,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
ResponseOutputMessage,
ResponseReasoningItem,
)
from openai.types.responses.response_output_item import ResponseApplyPatchToolCall
from litellm.types.utils import Choices, Message
@ -448,6 +449,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
accumulated_tool_calls.append(tool_call_dict)
tool_call_index += 1
elif isinstance(item, ResponseApplyPatchToolCall):
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
tool_call_dict = LiteLLMCompletionResponsesConfig.convert_apply_patch_tool_call_to_chat_completion_tool_call(
tool_call_item=item,
index=tool_call_index,
)
accumulated_tool_calls.append(tool_call_dict)
tool_call_index += 1
elif isinstance(item, dict) and handle_raw_dict_callback is not None:
# Handle raw dict responses (e.g., from GPT-5 Codex)
choice, index = handle_raw_dict_callback(item=item, index=index)
@ -1028,12 +1041,16 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
if provider_specific_fields:
tool_call_chunk.provider_specific_fields = provider_specific_fields # type: ignore
# Do NOT emit finish_reason here — response.completed handles the terminal
# finish_reason. Emitting "tool_calls" here would prematurely terminate
# the stream before subsequent tool calls arrive (same fix as #17246 for
# the message-type branch).
return ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(tool_calls=[tool_call_chunk]),
finish_reason="tool_calls",
delta=Delta(),
finish_reason=None,
)
]
)
@ -1091,6 +1108,12 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
finish_reason = "tool_calls" if has_function_calls else "stop"
usage = None
if response_data.get("usage"):
from litellm.responses.utils import ResponseAPILoggingUtils
usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
response_data.get("usage")
)
return ModelResponseStream(
choices=[
StreamingChoices(
@ -1098,7 +1121,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
delta=Delta(content=""),
finish_reason=finish_reason,
)
]
],
usage=usage
)
else:
pass

5
litellm/constants.py
View file

@ -1242,6 +1242,11 @@ X_LITELLM_DISABLE_CALLBACKS = "x-litellm-disable-callbacks"
LITELLM_METADATA_FIELD = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD = "metadata"
LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE = (
"Truncation is a DB storage safeguard. "
"Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.). "
"To increase the truncation limit, set `MAX_STRING_LENGTH_PROMPT_IN_DB` in your env."
)
########################### LiteLLM Proxy Specific Constants ###########################
########################################################################################

10
litellm/cost_calculator.py
View file

@ -272,6 +272,8 @@ def cost_per_token( # noqa: PLR0915
### SERVICE TIER ###
service_tier: Optional[str] = None, # for OpenAI service tier pricing
response: Optional[Any] = None,
### REQUEST MODEL ###
request_model: Optional[str] = None, # original request model for router detection
) -> Tuple[float, float]: # type: ignore
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -520,7 +522,7 @@ def cost_per_token( # noqa: PLR0915
return dashscope_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "azure_ai":
return azure_ai_cost_per_token(
model=model, usage=usage_block, response_time_ms=response_time_ms
model=model, usage=usage_block, response_time_ms=response_time_ms, request_model=request_model
)
else:
model_info = _cached_get_model_info_helper(
@ -1457,6 +1459,11 @@ def completion_cost( # noqa: PLR0915
text=completion_string
)
# Get the original request model for router detection
request_model_for_cost = None
if litellm_logging_obj is not None:
request_model_for_cost = litellm_logging_obj.model
(
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
@ -1479,6 +1486,7 @@ def completion_cost( # noqa: PLR0915
rerank_billed_units=rerank_billed_units,
service_tier=service_tier,
response=completion_response,
request_model=request_model_for_cost,
)
# Get additional costs from provider (e.g., routing fees, infrastructure costs)

343
litellm/files/main.py
View file

@ -7,7 +7,6 @@ https://platform.openai.com/docs/api-reference/files
import asyncio
import contextvars
import os
import time
import uuid as uuid_module
from functools import partial
@ -20,10 +19,12 @@ from litellm import get_secret_str
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.files.handler import AnthropicFilesHandler
from litellm.llms.azure.common_utils import get_azure_credentials
from litellm.llms.azure.files.handler import AzureOpenAIFilesAPI
from litellm.llms.bedrock.files.handler import BedrockFilesHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai.openai import FileDeleted, FileObject, OpenAIFilesAPI
from litellm.llms.vertex_ai.files.handler import VertexAIFilesHandler
from litellm.types.llms.openai import (
@ -185,95 +186,36 @@ def create_file(
timeout=timeout,
)
elif custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
or "https://api.openai.com/v1"
openai_creds = get_openai_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
organization=optional_params.organization,
)
organization = (
optional_params.organization
or litellm.organization
or os.getenv("OPENAI_ORGANIZATION", None)
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = (
optional_params.api_key
or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or os.getenv("OPENAI_API_KEY")
)
response = openai_files_instance.create_file(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_base=openai_creds.api_base,
api_key=openai_creds.api_key,
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
organization=openai_creds.organization,
create_file_data=_create_file_request,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_version = (
optional_params.api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
) # type: ignore
api_key = (
optional_params.api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
azure_creds = get_azure_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
api_version=optional_params.api_version,
)
response = azure_files_instance.create_file(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_version=api_version,
api_base=azure_creds.api_base,
api_key=azure_creds.api_key,
api_version=azure_creds.api_version,
timeout=timeout,
max_retries=optional_params.max_retries,
create_file_data=_create_file_request,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "vertex_ai":
api_base = optional_params.api_base or ""
vertex_ai_project = (
optional_params.vertex_project
or litellm.vertex_project
or get_secret_str("VERTEXAI_PROJECT")
)
vertex_ai_location = (
optional_params.vertex_location
or litellm.vertex_location
or get_secret_str("VERTEXAI_LOCATION")
)
vertex_credentials = optional_params.vertex_credentials or get_secret_str(
"VERTEXAI_CREDENTIALS"
)
response = vertex_ai_files_instance.create_file(
_is_async=_is_async,
api_base=api_base,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,
vertex_credentials=vertex_credentials,
timeout=timeout,
max_retries=optional_params.max_retries,
create_file_data=_create_file_request,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'create_file'. Only ['openai', 'azure', 'vertex_ai', 'manus'] are supported.".format(
@ -367,64 +309,31 @@ def file_retrieve(
_is_async = kwargs.pop("is_async", False) is True
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
or "https://api.openai.com/v1"
openai_creds = get_openai_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
organization=optional_params.organization,
)
organization = (
optional_params.organization
or litellm.organization
or os.getenv("OPENAI_ORGANIZATION", None)
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = (
optional_params.api_key
or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or os.getenv("OPENAI_API_KEY")
)
response = openai_files_instance.retrieve_file(
file_id=file_id,
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_base=openai_creds.api_base,
api_key=openai_creds.api_key,
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
organization=openai_creds.organization,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_version = (
optional_params.api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
) # type: ignore
api_key = (
optional_params.api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
azure_creds = get_azure_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
api_version=optional_params.api_version,
)
response = azure_files_instance.retrieve_file(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_version=api_version,
api_base=azure_creds.api_base,
api_key=azure_creds.api_key,
api_version=azure_creds.api_version,
timeout=timeout,
max_retries=optional_params.max_retries,
file_id=file_id,
@ -576,63 +485,31 @@ def file_delete(
timeout = 600.0
_is_async = kwargs.pop("is_async", False) is True
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
or "https://api.openai.com/v1"
)
organization = (
optional_params.organization
or litellm.organization
or os.getenv("OPENAI_ORGANIZATION", None)
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = (
optional_params.api_key
or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or os.getenv("OPENAI_API_KEY")
openai_creds = get_openai_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
organization=optional_params.organization,
)
response = openai_files_instance.delete_file(
file_id=file_id,
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_base=openai_creds.api_base,
api_key=openai_creds.api_key,
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
organization=openai_creds.organization,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_version = (
optional_params.api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
) # type: ignore
api_key = (
optional_params.api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
azure_creds = get_azure_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
api_version=optional_params.api_version,
)
response = azure_files_instance.delete_file(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_version=api_version,
api_base=azure_creds.api_base,
api_key=azure_creds.api_key,
api_version=azure_creds.api_version,
timeout=timeout,
max_retries=optional_params.max_retries,
file_id=file_id,
@ -815,64 +692,31 @@ def file_list(
)
return response
elif custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
or "https://api.openai.com/v1"
openai_creds = get_openai_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
organization=optional_params.organization,
)
organization = (
optional_params.organization
or litellm.organization
or os.getenv("OPENAI_ORGANIZATION", None)
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = (
optional_params.api_key
or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or os.getenv("OPENAI_API_KEY")
)
response = openai_files_instance.list_files(
purpose=purpose,
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_base=openai_creds.api_base,
api_key=openai_creds.api_key,
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
organization=openai_creds.organization,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_version = (
optional_params.api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
) # type: ignore
api_key = (
optional_params.api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
azure_creds = get_azure_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
api_version=optional_params.api_version,
)
response = azure_files_instance.list_files(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_version=api_version,
api_base=azure_creds.api_base,
api_key=azure_creds.api_key,
api_version=azure_creds.api_version,
timeout=timeout,
max_retries=optional_params.max_retries,
purpose=purpose,
@ -1003,64 +847,31 @@ def file_content(
return response
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
or "https://api.openai.com/v1"
openai_creds = get_openai_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
organization=optional_params.organization,
)
organization = (
optional_params.organization
or litellm.organization
or os.getenv("OPENAI_ORGANIZATION", None)
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = (
optional_params.api_key
or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or os.getenv("OPENAI_API_KEY")
)
response = openai_files_instance.file_content(
_is_async=_is_async,
file_content_request=_file_content_request,
api_base=api_base,
api_key=api_key,
api_base=openai_creds.api_base,
api_key=openai_creds.api_key,
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
organization=openai_creds.organization,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_version = (
optional_params.api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
) # type: ignore
api_key = (
optional_params.api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
azure_creds = get_azure_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
api_version=optional_params.api_version,
)
response = azure_files_instance.file_content(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_version=api_version,
api_base=azure_creds.api_base,
api_key=azure_creds.api_key,
api_version=azure_creds.api_version,
timeout=timeout,
max_retries=optional_params.max_retries,
file_content_request=_file_content_request,

89
litellm/fine_tuning/main.py
View file

@ -126,6 +126,31 @@ async def acreate_fine_tuning_job(
raise e
def _build_fine_tuning_job_data(model, training_file, hyperparameters, suffix, validation_file, integrations, seed):
return FineTuningJobCreate(
model=model,
training_file=training_file,
hyperparameters=hyperparameters,
suffix=suffix,
validation_file=validation_file,
integrations=integrations,
seed=seed,
)
def _resolve_fine_tuning_timeout(
timeout: Any,
custom_llm_provider: str,
) -> Union[float, httpx.Timeout]:
"""Normalise a raw timeout value to a float (seconds) or httpx.Timeout for fine-tuning calls."""
timeout = timeout or 600.0
if isinstance(timeout, httpx.Timeout):
if not supports_httpx_timeout(custom_llm_provider):
return float(timeout.read or 600)
return timeout
return float(timeout)
@client
def create_fine_tuning_job(
model: str,
@ -164,21 +189,10 @@ def create_fine_tuning_job(
_oai_hyperparameters: Hyperparameters = Hyperparameters(
**hyperparameters
) # Typed Hyperparameters for OpenAI Spec
### TIMEOUT LOGIC ###
timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
# set timeout for 10 minutes by default
if (
timeout is not None
and isinstance(timeout, httpx.Timeout)
and supports_httpx_timeout(custom_llm_provider) is False
):
read_timeout = timeout.read or 600
timeout = read_timeout # default 10 min timeout
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
timeout = float(timeout) # type: ignore
elif timeout is None:
timeout = 600.0
timeout = _resolve_fine_tuning_timeout(
optional_params.timeout or kwargs.get("request_timeout", 600),
custom_llm_provider,
)
# OpenAI
if custom_llm_provider == "openai":
@ -204,19 +218,9 @@ def create_fine_tuning_job(
or os.getenv("OPENAI_API_KEY")
)
create_fine_tuning_job_data = FineTuningJobCreate(
model=model,
training_file=training_file,
hyperparameters=_oai_hyperparameters,
suffix=suffix,
validation_file=validation_file,
integrations=integrations,
seed=seed,
)
create_fine_tuning_job_data_dict = create_fine_tuning_job_data.model_dump(
exclude_none=True
)
create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
model, training_file, _oai_hyperparameters, suffix, validation_file, integrations, seed,
).model_dump(exclude_none=True)
response = openai_fine_tuning_apis_instance.create_fine_tuning_job(
api_base=api_base,
@ -258,20 +262,10 @@ def create_fine_tuning_job(
# Prepare Azure-specific parameters for extra_body
extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams)
create_fine_tuning_job_data = FineTuningJobCreate(
model=model,
training_file=training_file,
hyperparameters=_oai_hyperparameters,
suffix=suffix,
validation_file=validation_file,
integrations=integrations,
seed=seed,
)
create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
model, training_file, _oai_hyperparameters, suffix, validation_file, integrations, seed,
).model_dump(exclude_none=True)
create_fine_tuning_job_data_dict = create_fine_tuning_job_data.model_dump(
exclude_none=True
)
# Add extra_body if it has Azure-specific parameters
if extra_body:
create_fine_tuning_job_data_dict["extra_body"] = extra_body
@ -301,18 +295,11 @@ def create_fine_tuning_job(
vertex_credentials = optional_params.vertex_credentials or get_secret_str(
"VERTEXAI_CREDENTIALS"
)
create_fine_tuning_job_data = FineTuningJobCreate(
model=model,
training_file=training_file,
hyperparameters=_oai_hyperparameters,
suffix=suffix,
validation_file=validation_file,
integrations=integrations,
seed=seed,
)
response = vertex_fine_tuning_apis_instance.create_fine_tuning_job(
_is_async=_is_async,
create_fine_tuning_job_data=create_fine_tuning_job_data,
create_fine_tuning_job_data=_build_fine_tuning_job_data(
model, training_file, _oai_hyperparameters, suffix, validation_file, integrations, seed,
),
vertex_credentials=vertex_credentials,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,

16
litellm/integrations/custom_guardrail.py
View file

@ -231,8 +231,14 @@ class CustomGuardrail(CustomLogger):
event_hook, supported_event_hooks
)
elif isinstance(event_hook, Mode):
tag_values_flat: list = []
for v in event_hook.tags.values():
if isinstance(v, list):
tag_values_flat.extend(v)
else:
tag_values_flat.append(v)
_validate_event_hook_list_is_in_supported_event_hooks(
list(event_hook.tags.values()), supported_event_hooks
tag_values_flat, supported_event_hooks
)
if event_hook.default:
default_list = (
@ -466,8 +472,12 @@ class CustomGuardrail(CustomLogger):
if isinstance(self.event_hook, list):
return event_type.value in self.event_hook
if isinstance(self.event_hook, Mode):
if event_type.value in self.event_hook.tags.values():
return True
for tag_value in self.event_hook.tags.values():
if isinstance(tag_value, list):
if event_type.value in tag_value:
return True
elif event_type.value == tag_value:
return True
if self.event_hook.default:
default_list = (
self.event_hook.default

6
litellm/integrations/helicone.py
View file

@ -167,12 +167,12 @@ class HeliconeLogger:
if "claude" in model and not is_vertex_ai:
url = f"{self.api_base}/anthropic/v1/log"
provider_url = "https://api.anthropic.com/v1/messages"
elif "gemini" in model:
url = f"{self.api_base}/custom/v1/log"
provider_url = "https://generativelanguage.googleapis.com/v1beta"
elif is_vertex_ai:
url = f"{self.api_base}/custom/v1/log"
provider_url = "https://aiplatform.googleapis.com/v1"
elif "gemini" in model:
url = f"{self.api_base}/custom/v1/log"
provider_url = "https://generativelanguage.googleapis.com/v1beta"
headers = {
"Authorization": f"Bearer {self.key}",
"Content-Type": "application/json",

68
litellm/integrations/opentelemetry.py
View file

@ -735,13 +735,10 @@ class OpenTelemetry(CustomLogger):
self._maybe_log_raw_request(
kwargs, response_obj, start_time, end_time, span
)
# Ensure proxy-request parent span is annotated with the actual operation kind
if (
parent_span is not None
and hasattr(parent_span, "name")
and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
):
self.set_attributes(parent_span, kwargs, response_obj)
# Do NOT duplicate attributes onto the parent proxy-request span.
# The child litellm_request span already carries all attributes;
# copying them to the parent doubles storage and complicates
# search (Issue #4).
else:
# Do not create primary span (keep hierarchy shallow when parent exists)
from opentelemetry.trace import Status, StatusCode
@ -757,8 +754,12 @@ class OpenTelemetry(CustomLogger):
kwargs, response_obj, start_time, end_time, parent_span
)
# 3. Guardrail span
self._create_guardrail_span(kwargs=kwargs, context=ctx)
# 3. Guardrail span — ensure guardrails are always parented to an
# existing span so they never become orphaned root spans (Issue #5).
guardrail_ctx = self._resolve_guardrail_context(
span=span, parent_span=parent_span, fallback_ctx=ctx
)
self._create_guardrail_span(kwargs=kwargs, context=guardrail_ctx)
# 4. Metrics & cost recording
self._record_metrics(kwargs, response_obj, start_time, end_time)
@ -1145,6 +1146,27 @@ class OpenTelemetry(CustomLogger):
)
otel_logger.emit(log_record)
@staticmethod
def _resolve_guardrail_context(
span: Optional[Any],
parent_span: Optional[Any],
fallback_ctx: Optional[Any],
) -> Optional[Any]:
"""
Return a valid OTEL context for guardrail child spans so they are
never orphaned (Issue #5). Priority:
1. The litellm_request span that was just created
2. The parent proxy-request span
3. The original fallback context (may be None — last resort)
"""
from opentelemetry import trace as _trace
if span is not None:
return _trace.set_span_in_context(span)
if parent_span is not None:
return _trace.set_span_in_context(parent_span)
return fallback_ctx
def _create_guardrail_span(
self, kwargs: Optional[dict], context: Optional[Context]
):
@ -1250,6 +1272,7 @@ class OpenTelemetry(CustomLogger):
"USE_OTEL_LITELLM_REQUEST_SPAN"
)
span = None
if should_create_primary_span:
# Span 1: Request sent to litellm SDK
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
@ -1275,8 +1298,11 @@ class OpenTelemetry(CustomLogger):
self.set_attributes(parent_otel_span, kwargs, response_obj)
self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
# Create span for guardrail information
self._create_guardrail_span(kwargs=kwargs, context=_parent_context)
# Create span for guardrail information — ensure proper parenting (Issue #5)
guardrail_ctx = self._resolve_guardrail_context(
span=span, parent_span=parent_otel_span, fallback_ctx=_parent_context
)
self._create_guardrail_span(kwargs=kwargs, context=guardrail_ctx)
# Do NOT end parent span - it should be managed by its creator
# External spans (from Langfuse, user code, HTTP headers, global context) must not be closed by LiteLLM
@ -1579,12 +1605,20 @@ class OpenTelemetry(CustomLogger):
value=optional_params.get("user"),
)
# The unique identifier for the completion.
if response_obj and response_obj.get("id"):
# The unique identifier for the LLM call.
# Completions have a provider response ID (e.g. "chatcmpl-xxx"),
# but Embeddings and Image-gen responses do not. Fall back to
# the litellm call ID so every call type can be correlated
# across LiteLLM UI, Phoenix traces, and provider logs (Issue #8).
response_id = (
(response_obj.get("id") if response_obj else None)
or standard_logging_payload.get("id")
)
if response_id:
self.safe_set_attribute(
span=span,
key="gen_ai.response.id",
value=response_obj.get("id"),
value=response_id,
)
# The model used to generate the response.
@ -1808,8 +1842,10 @@ class OpenTelemetry(CustomLogger):
def set_raw_request_attributes(self, span: Span, kwargs, response_obj):
try:
self.set_attributes(span, kwargs, response_obj)
kwargs.get("optional_params", {})
# Only set provider-specific raw payload attributes on this span.
# The parent litellm_request span already carries the standard
# gen_ai.* / metadata.* attributes — duplicating them here doubles
# storage and adds noise (Issue #3).
litellm_params = kwargs.get("litellm_params", {}) or {}
custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown")

91
litellm/integrations/websearch_interception/handler.py
View file

@ -7,6 +7,7 @@ server-side using litellm router's search tools.
"""
import asyncio
import math
from typing import Any, Dict, List, Optional, Tuple, Union, cast
import litellm
@ -481,6 +482,56 @@ class WebSearchInterceptionLogger(CustomLogger):
response_format=response_format,
)
@staticmethod
def _resolve_max_tokens(
optional_params: Dict,
kwargs: Dict,
) -> int:
"""Extract max_tokens and validate against thinking.budget_tokens.
Anthropic API requires ``max_tokens > thinking.budget_tokens``.
If the constraint is violated, auto-adjust to ``budget_tokens + 1024``.
"""
max_tokens: int = optional_params.get(
"max_tokens",
kwargs.get("max_tokens", 1024),
)
thinking_param = optional_params.get("thinking")
if thinking_param and isinstance(thinking_param, dict):
budget_tokens = thinking_param.get("budget_tokens")
if (
budget_tokens is not None
and isinstance(budget_tokens, (int, float))
and math.isfinite(budget_tokens)
and budget_tokens > 0
):
if max_tokens <= budget_tokens:
adjusted = math.ceil(budget_tokens) + 1024
verbose_logger.debug(
"WebSearchInterception: max_tokens=%s <= thinking.budget_tokens=%s, "
"adjusting to %s to satisfy Anthropic API constraint",
max_tokens, budget_tokens, adjusted,
)
max_tokens = adjusted
return max_tokens
@staticmethod
def _prepare_followup_kwargs(kwargs: Dict) -> Dict:
"""Build kwargs for the follow-up call, excluding internal keys.
``litellm_logging_obj`` MUST be excluded so the follow-up call creates
its own ``Logging`` instance via ``function_setup``. Reusing the
initial call's logging object triggers the dedup flag
(``has_logged_async_success``) which silently prevents the initial
call's spend from being recorded — the root cause of the
SpendLog / AWS billing mismatch.
"""
_internal_keys = {'litellm_logging_obj'}
return {
k: v for k, v in kwargs.items()
if not k.startswith('_websearch_interception') and k not in _internal_keys
}
async def _execute_agentic_loop(
self,
model: str,
@ -504,7 +555,7 @@ class WebSearchInterceptionLogger(CustomLogger):
)
search_tasks.append(self._execute_search(query))
else:
verbose_logger.warning(
verbose_logger.debug(
f"WebSearchInterception: Tool call {tool_call['id']} has no query"
)
# Add empty result for tools without query
@ -531,7 +582,7 @@ class WebSearchInterceptionLogger(CustomLogger):
final_search_results.append(cast(str, result))
else:
# Should never happen, but handle for type safety
verbose_logger.warning(
verbose_logger.debug(
f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
)
final_search_results.append(str(result))
@ -557,13 +608,18 @@ class WebSearchInterceptionLogger(CustomLogger):
f"WebSearchInterception: Last message (tool_result): {user_message}"
)
# Correlation context for structured logging
_call_id = (
getattr(logging_obj, "litellm_call_id", None)
or kwargs.get("litellm_call_id", "unknown")
)
full_model_name = model # safe default before try block
# Use anthropic_messages.acreate for follow-up request
try:
# Extract max_tokens from optional params or kwargs
# max_tokens is a required parameter for anthropic_messages.acreate()
max_tokens = anthropic_messages_optional_request_params.get(
"max_tokens",
kwargs.get("max_tokens", 1024) # Default to 1024 if not found
max_tokens = self._resolve_max_tokens(
anthropic_messages_optional_request_params, kwargs
)
verbose_logger.debug(
@ -576,16 +632,10 @@ class WebSearchInterceptionLogger(CustomLogger):
if k != 'max_tokens'
}
# Remove internal websearch interception flags from kwargs before follow-up request
# These flags are used internally and should not be passed to the LLM provider
kwargs_for_followup = {
k: v for k, v in kwargs.items()
if not k.startswith('_websearch_interception')
}
kwargs_for_followup = self._prepare_followup_kwargs(kwargs)
# Get model from logging_obj.model_call_details["agentic_loop_params"]
# This preserves the full model name with provider prefix (e.g., "bedrock/invoke/...")
full_model_name = model
if logging_obj is not None:
agentic_params = logging_obj.model_call_details.get("agentic_loop_params", {})
full_model_name = agentic_params.get("model", model)
@ -609,7 +659,10 @@ class WebSearchInterceptionLogger(CustomLogger):
return final_response
except Exception as e:
verbose_logger.exception(
f"WebSearchInterception: Follow-up request failed: {str(e)}"
"WebSearchInterception: Follow-up request failed "
"[call_id=%s model=%s messages=%d searches=%d]: %s",
_call_id, full_model_name, len(follow_up_messages),
len(final_search_results), str(e),
)
raise
@ -620,7 +673,7 @@ class WebSearchInterceptionLogger(CustomLogger):
try:
from litellm.proxy.proxy_server import llm_router
except ImportError:
verbose_logger.warning(
verbose_logger.debug(
"WebSearchInterception: Could not import llm_router from proxy_server, "
"falling back to direct litellm.asearch() with perplexity"
)
@ -643,7 +696,7 @@ class WebSearchInterceptionLogger(CustomLogger):
f"with provider '{search_provider}'"
)
else:
verbose_logger.warning(
verbose_logger.debug(
f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in router, "
"falling back to first available or perplexity"
)
@ -717,7 +770,7 @@ class WebSearchInterceptionLogger(CustomLogger):
)
search_tasks.append(self._execute_search(query))
else:
verbose_logger.warning(
verbose_logger.debug(
f"WebSearchInterception: Tool call {tool_call.get('id')} has no query"
)
# Add empty result for tools without query
@ -742,7 +795,7 @@ class WebSearchInterceptionLogger(CustomLogger):
elif isinstance(result, str):
final_search_results.append(cast(str, result))
else:
verbose_logger.warning(
verbose_logger.debug(
f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
)
final_search_results.append(str(result))

10
litellm/litellm_core_utils/duration_parser.py
View file

@ -64,12 +64,10 @@ def duration_in_seconds(duration: str) -> int:
now = time.time()
current_time = datetime.fromtimestamp(now)
if current_time.month == 12:
target_year = current_time.year + 1
target_month = 1
else:
target_year = current_time.year
target_month = current_time.month + value
# Calculate target month and year, handling overflow past December
total_months = current_time.month - 1 + value # 0-indexed months
target_year = current_time.year + total_months // 12
target_month = total_months % 12 + 1 # back to 1-indexed
# Determine the day to set for next month
target_day = current_time.day

7
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -561,6 +561,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
) = litellm.GroqChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "bedrock_mantle":
(
api_base,
dynamic_api_key,
) = litellm.BedrockMantleChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "nvidia_nim":
# nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
api_base = (

2
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -88,6 +88,8 @@ def get_supported_openai_params( # noqa: PLR0915
return litellm.VolcEngineConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "groq":
return litellm.GroqChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "bedrock_mantle":
return litellm.BedrockMantleChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "hosted_vllm":
return litellm.HostedVLLMChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "vllm":

27
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -2,7 +2,7 @@ import asyncio
import json
import time
import traceback
from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union
from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union, cast
import litellm
from litellm._logging import verbose_logger
@ -13,6 +13,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.types.llms.databricks import DatabricksTool
from litellm.types.llms.openai import (
ChatCompletionThinkingBlock,
ImageURLListItem,
OpenAIModerationResponse,
)
from litellm.types.utils import (
@ -26,13 +27,13 @@ from litellm.types.utils import (
Function,
HiddenParams,
ImageResponse,
PromptTokensDetailsWrapper,
)
from litellm.types.utils import Logprobs as TextCompletionLogprobs
from litellm.types.utils import (
Message,
ModelResponse,
ModelResponseStream,
PromptTokensDetailsWrapper,
RerankResponse,
StreamingChoices,
TextChoices,
@ -52,6 +53,24 @@ _MODEL_RESPONSE_FIELDS: frozenset = frozenset(ModelResponse.model_fields.keys())
}
def _normalize_images_for_message(
images: Optional[List[dict]],
) -> Optional[List[ImageURLListItem]]:
"""
Ensure each image has an 'index' field, as required by ImageURLListItem.
Some providers (e.g. OpenRouter) return images without index.
"""
if not images:
return cast(Optional[List[ImageURLListItem]], images)
normalized: List[ImageURLListItem] = []
for i, img in enumerate(images):
if isinstance(img, dict) and "index" not in img:
normalized.append(cast(ImageURLListItem, {**img, "index": i}))
else:
normalized.append(cast(ImageURLListItem, img))
return normalized
def _safe_convert_created_field(created_value) -> int:
"""
Safely convert a 'created' field value to an integer.
@ -591,7 +610,9 @@ def convert_to_model_response_object( # noqa: PLR0915
reasoning_content=reasoning_content,
thinking_blocks=thinking_blocks,
annotations=choice["message"].get("annotations", None),
images=choice["message"].get("images", None),
images=_normalize_images_for_message(
choice["message"].get("images", None)
),
)
finish_reason = choice.get("finish_reason", None)
if finish_reason is None:

95
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -20,6 +20,7 @@ from typing import (
cast,
)
from litellm import verbose_logger
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
@ -1278,16 +1279,76 @@ def extract_images_from_message(message: AllMessageValues) -> List[str]:
return images
def _attempt_json_repair(s: str) -> Optional[Any]:
"""
Attempt to repair truncated JSON produced by LLM tool calls.
Handles the most common truncation patterns where the model generates
valid JSON that is cut short (missing closing brackets/braces).
Returns the parsed value on success, or None if repair fails.
"""
import json
stripped = s.rstrip()
if not stripped:
return None
# Track the stack of unmatched openers to respect nesting order
opener_stack: list = []
in_string = False
escape_next = False
for ch in stripped:
if escape_next:
escape_next = False
continue
if ch == "\\":
if in_string:
escape_next = True
continue
if ch == '"':
in_string = not in_string
continue
if in_string:
continue
if ch == "{":
opener_stack.append("}")
elif ch == "[":
opener_stack.append("]")
elif ch in ("}", "]"):
if opener_stack and opener_stack[-1] == ch:
opener_stack.pop()
if not opener_stack:
return None
# Remove trailing comma before we close brackets
candidate = stripped.rstrip(",")
# Close in reverse order of opening (respects nesting)
candidate += "".join(reversed(opener_stack))
try:
return json.loads(candidate)
except json.JSONDecodeError:
pass
return None
def parse_tool_call_arguments(
arguments: Optional[str],
tool_name: Optional[str] = None,
context: Optional[str] = None,
) -> Dict[str, Any]:
) -> Any:
"""
Parse tool call arguments from a JSON string.
This function handles malformed JSON gracefully by raising a ValueError
with context about what failed and what the problematic input was.
When the JSON is malformed (e.g. truncated by the model), this function
attempts a lightweight repair (closing unmatched brackets/braces) before
raising an error. A warning is logged whenever repair succeeds so that
callers are aware the arguments were not perfectly formed.
Args:
arguments: The JSON string containing tool arguments, or None.
@ -1295,19 +1356,34 @@ def parse_tool_call_arguments(
context: Optional context string (e.g., "Anthropic Messages API").
Returns:
Parsed arguments as a dictionary. Returns empty dict if arguments is None or empty.
Parsed arguments (usually a dict, but may be any JSON-deserializable
type such as list, str, int, float, or None). Returns empty dict if
arguments is None or empty.
Raises:
ValueError: If the arguments string is not valid JSON.
ValueError: If the arguments string is not valid JSON and cannot be repaired.
"""
import json
if not arguments:
if not arguments or not arguments.strip():
return {}
try:
return json.loads(arguments)
except json.JSONDecodeError as e:
except json.JSONDecodeError as original_error:
repaired = _attempt_json_repair(arguments)
if repaired is not None:
verbose_logger.warning(
"Repaired truncated tool call arguments for tool '%s' (%s). "
"Original (%d chars): %.200s%s",
tool_name or "<unknown>",
context or "unknown context",
len(arguments),
arguments,
"..." if len(arguments) > 200 else "",
)
return repaired
error_parts = ["Failed to parse tool call arguments"]
if tool_name:
@ -1316,10 +1392,11 @@ def parse_tool_call_arguments(
error_parts.append(f"({context})")
error_message = (
" ".join(error_parts) + f". Error: {str(e)}. Arguments: {arguments}"
" ".join(error_parts)
+ f". Error: {str(original_error)}. Arguments: {arguments}"
)
raise ValueError(error_message) from e
raise ValueError(error_message) from original_error
def split_concatenated_json_objects(raw: str) -> List[Dict[str, Any]]:

10
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1035,9 +1035,13 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str:
parsed_args = parse_tool_call_arguments(
tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke"
)
parameters = "".join(
f"<{param}>{val}</{param}>\n" for param, val in parsed_args.items()
)
if isinstance(parsed_args, dict):
parameters = "".join(
f"<{param}>{val}</{param}>\n"
for param, val in parsed_args.items()
)
else:
parameters = f"<result>{parsed_args}</result>\n"
invokes += (
"<invoke>\n"
f"<tool_name>{tool_name}</tool_name>\n"

70
litellm/litellm_core_utils/redact_messages.py
View file

@ -73,6 +73,53 @@ def _redact_responses_api_output(output_items):
summary_item.text = "redacted-by-litellm"
def _redact_standard_logging_object(model_call_details: dict):
"""Redact messages and response inside standard_logging_object if present."""
standard_logging_object = model_call_details.get("standard_logging_object")
if standard_logging_object is None:
return
redacted_str = "redacted-by-litellm"
if standard_logging_object.get("messages") is not None:
standard_logging_object["messages"] = [
{"role": "user", "content": redacted_str}
]
response = standard_logging_object.get("response")
if response is not None:
if isinstance(response, dict) and "output" in response:
# ResponsesAPIResponse format - redact content in output items
if isinstance(response.get("output"), list):
for output_item in response["output"]:
if isinstance(output_item, dict) and "content" in output_item:
if isinstance(output_item["content"], list):
for content_item in output_item["content"]:
if (
isinstance(content_item, dict)
and "text" in content_item
):
content_item["text"] = redacted_str
elif isinstance(response, dict) and "choices" in response:
# ModelResponse dict format - redact content in choices
if isinstance(response.get("choices"), list):
for choice in response["choices"]:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = redacted_str
if "audio" in choice["message"]:
choice["message"]["audio"] = None
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = redacted_str
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
elif isinstance(response, str):
standard_logging_object["response"] = redacted_str
else:
# For other formats (empty dict, None, etc.), use simple text format
standard_logging_object["response"] = {"text": redacted_str}
def perform_redaction(model_call_details: dict, result):
"""
Performs the actual redaction on the logging object and result.
@ -114,6 +161,29 @@ def perform_redaction(model_call_details: dict, result):
if hasattr(_result, "choices") and _result.choices is not None:
for choice in _result.choices:
_redact_choice_content(choice)
elif isinstance(_result, dict) and "choices" in _result:
# Handle dict representation of ModelResponse (e.g., from model_dump())
if _result.get("choices") is not None:
for choice in _result["choices"]:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = "redacted-by-litellm"
if "reasoning_content" in choice["message"]:
choice["message"]["reasoning_content"] = "redacted-by-litellm"
if "thinking_blocks" in choice["message"]:
choice["message"]["thinking_blocks"] = None
if "audio" in choice["message"]:
choice["message"]["audio"] = None
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = "redacted-by-litellm"
if "reasoning_content" in choice["delta"]:
choice["delta"]["reasoning_content"] = "redacted-by-litellm"
if "thinking_blocks" in choice["delta"]:
choice["delta"]["thinking_blocks"] = None
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
else:
_redact_choice_content(choice)
elif isinstance(_result, litellm.ResponsesAPIResponse):
if hasattr(_result, "output"):
_redact_responses_api_output(_result.output)

6
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -476,13 +476,15 @@ class ChunkProcessor:
"prompt_tokens_details": prompt_tokens_details,
}
def count_reasoning_tokens(self, response: ModelResponse) -> int:
reasoning_tokens = 0
def count_reasoning_tokens(self, response: ModelResponse) -> Optional[int]:
reasoning_tokens: Optional[int] = None
for choice in response.choices:
if (
hasattr(cast(Choices, choice).message, "reasoning_content")
and cast(Choices, choice).message.reasoning_content is not None
):
if reasoning_tokens is None:
reasoning_tokens = 0
reasoning_tokens += token_counter(
text=cast(Choices, choice).message.reasoning_content,
count_response_tokens=True,

9
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1099,7 +1099,14 @@ class CustomStreamWrapper:
and self.custom_llm_provider in litellm._custom_providers
):
if self.received_finish_reason is not None:
if "provider_specific_fields" not in chunk:
_chunk_has_content = isinstance(chunk, dict) and (
bool(chunk.get("text", ""))
or chunk.get("tool_use") is not None
)
if not _chunk_has_content and (
not isinstance(chunk, dict)
or "provider_specific_fields" not in chunk
):
raise StopIteration
anthropic_response_obj: GChunk = cast(GChunk, chunk)
completion_obj["content"] = anthropic_response_obj["text"]

Some files were not shown because too many files have changed in this diff Show more