scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

Merge main and fix Greptile feedback

Browse source 
- Resolve merge conflicts with main (new agent spend tracking, tool discovery, etc.)
- Move urllib imports to module level in proxy_server.py (P2 feedback)
- Replace conditional `if` guards with unconditional `assert` in redirect tests (P1 feedback)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
yuneng-jiang 2026-03-17 16:06:58 -07:00
commit b9bfd05150
2140 changed files with 183462 additions and 28936 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

195
.circleci/config.yml
View file

@ -69,9 +69,11 @@ jobs:
- run:
name: Install Python
command: |
choco install python --version=3.11.0 -y
choco install python --version=3.11.0 -y --no-progress --force
refreshenv
python --version
environment:
CHOCOLATEY_CONFIRM_ALL: "true"
- run:
name: Install Dependencies
command: |
@ -1181,7 +1183,7 @@ jobs:
command: |
pwd
ls
python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO
python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 4 --timeout=300 -vv --log-cli-level=INFO
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -1699,7 +1701,7 @@ jobs:
command: |
prisma generate
export PYTHONUNBUFFERED=1
python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A
no_output_timeout: 60m
- run:
name: Rename the coverage files
@ -3689,6 +3691,114 @@ jobs:
- store_test_results:
path: test-results
proxy_e2e_azure_batches_tests:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Docker CLI
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.12
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.12 -y
conda activate myenv
python --version
- run:
name: Install Poetry
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
pip install poetry
- run:
name: Install dockerize
command: |
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=llmproxy \
-e POSTGRES_PASSWORD=dbpassword9090 \
-e POSTGRES_DB=litellm \
-p 5432:5432 \
postgres:15
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Install system dependencies
command: |
sudo apt-get update -y
sudo apt-get install -y libpq-dev
- run:
name: Install Dependencies
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry config virtualenvs.in-project true
poetry install --with dev,proxy-dev --extras "proxy"
poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
- run:
name: Setup litellm-enterprise
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry run pip install --force-reinstall --no-deps -e enterprise/
- run:
name: Generate Prisma client
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry run prisma generate --schema litellm/proxy/schema.prisma
- run:
name: Run Prisma migrations
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
cd litellm/proxy
poetry run prisma migrate deploy --schema schema.prisma
cd ../..
- run:
name: Run Azure Batch E2E Tests
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
export USE_LOCAL_LITELLM=true
export USE_MOCK_MODELS=true
export USE_STATE_TRACKER=true
export LITELLM_LOG=DEBUG
poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
-vv -s -k "test_e2e_managed_batch" \
--tb=short \
--maxfail=3 \
--durations=10 \
--junitxml=test-results/junit.xml
no_output_timeout: 30m
upload-coverage:
docker:
- image: cimg/python:3.9
@ -3886,7 +3996,7 @@ jobs:
command: |
cd ~/project
# Check pyproject.toml
CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras'].split('\"')[1])")
CURRENT_VERSION=$(python -c "import toml; dep = toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras']; print(dep['version'] if isinstance(dep, dict) else dep)")
if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)"
exit 1
@ -4100,6 +4210,63 @@ jobs:
path: playwright-report
destination: playwright-report
prisma_schema_sync:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Install Neon CLI
command: |
npm i -g neonctl
- run:
name: Install curl and dockerize
command: |
sudo apt-get update
sudo apt-get install -y curl
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Sync schema on base e2e database
command: |
BASE_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch br-fancy-paper-ad1olsb3 \
--database-name yuneng-trial-db \
--role neondb_owner)
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$BASE_DATABASE_URL \
-e LITELLM_MASTER_KEY="sk-1234" \
--name schema-sync \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--use_prisma_db_push
- run:
name: Start outputting logs
command: docker logs -f schema-sync
background: true
- run:
name: Wait for proxy to be ready (schema sync complete)
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Stop schema sync container
command: docker stop schema-sync
test_nonroot_image:
machine:
image: ubuntu-2204:2023.10.1
@ -4170,7 +4337,8 @@ jobs:
name: Check for expected error
command: |
if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
grep -q "ERROR: Application startup failed. Exiting." docker_output.log; then
(grep -q "Database setup failed after multiple retries" docker_output.log || \
grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then
echo "Expected error found. Test passed."
else
echo "Expected error not found. Test failed."
@ -4298,6 +4466,15 @@ workflows:
only:
- main
- /litellm_.*/
- prisma_schema_sync:
context: e2e_ui_tests
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- e2e_ui_testing:
name: e2e_ui_testing_chromium
browser: chromium
@ -4305,6 +4482,7 @@ workflows:
requires:
- ui_build
- build_docker_database_image
- prisma_schema_sync
filters:
branches:
only:
@ -4317,6 +4495,7 @@ workflows:
requires:
- ui_build
- build_docker_database_image
- prisma_schema_sync
filters:
branches:
only:
@ -4390,6 +4569,12 @@ workflows:
only:
- main
- /litellm_.*/
- proxy_e2e_azure_batches_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- llm_translation_testing:
filters:
branches:

2
.github/ISSUE_TEMPLATE/config.yml vendored
View file

@ -1,7 +1,7 @@
blank_issues_enabled: true
contact_links:
- name: Schedule Demo
url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat
url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions
about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM
- name: Discord
url: https://discord.com/invite/wuPM9dRgDw

22
.github/codeql/codeql-config.yml vendored Normal file
View file

@ -0,0 +1,22 @@
name: "LiteLLM CodeQL config"
# Use security-extended suite instead of security-and-quality to avoid
# result sets > 2 GiB on this codebase that cause fatal OOM failures.
queries:
- uses: security-extended
# These two queries are security queries included in security-extended that
# individually produce result sets > 2 GiB on this codebase, causing fatal
# OOM failures. Exclude them as a safety net until CI confirms they no longer
# OOM; drop these exclusions in a follow-up once verified.
query-filters:
- exclude:
id: py/clear-text-logging-sensitive-data # CWE-312 — > 2 GiB result set
- exclude:
id: py/polynomial-redos # CWE-730 — > 2 GiB result set
paths-ignore:
- tests
- docs
- "**/*.md"
- litellm/proxy/_experimental/out

19
.github/observatory/litellm_config.yaml vendored Normal file
View file

@ -0,0 +1,19 @@
# LiteLLM Observatory Test Configuration
# This config is used by CI to spin up a temporary LiteLLM instance
# for running observatory tests against RC/stable releases.
#
# Add model definitions for the providers you want to test.
# Provider API keys are injected via environment variables in CI.
model_list:
- model_name: gpt-4o
litellm_params:
model: azure/gpt-4o
api_key: os.environ/AZURE_API_KEY
api_base: os.environ/AZURE_API_BASE
- model_name: gpt-4o-mini
litellm_params:
model: azure/gpt-4o-mini
api_key: os.environ/AZURE_API_KEY
api_base: os.environ/AZURE_API_BASE

2
.github/pull_request_template.md vendored
View file

@ -6,7 +6,7 @@
**Please complete all items before asking a LiteLLM maintainer to review your PR**
- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review

208
.github/scripts/close_duplicate_issues.py vendored Executable file
View file

@ -0,0 +1,208 @@
#!/usr/bin/env python3
"""
Detect and close duplicate GitHub issues using title similarity.
Modes:
--scan Compare all open issues against each other (batch)
--issue-number N Check a single issue against older open issues
Requires the `gh` CLI to be authenticated.
"""
import argparse
import difflib
import json
import re
import subprocess
import sys
def normalize_title(title: str) -> str:
"""Strip common prefixes, lowercase, and collapse whitespace."""
title = re.sub(
r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
"",
title,
flags=re.IGNORECASE,
)
return " ".join(title.lower().split())
def gh(*args: str) -> str:
"""Run a gh CLI command and return stdout."""
result = subprocess.run(
["gh", *args],
capture_output=True,
text=True,
check=True,
)
return result.stdout
def fetch_open_issues(repo: str | None) -> list[dict]:
"""Fetch all open issues (excluding PRs) via gh api --paginate."""
if repo:
endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
else:
endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
cmd = ["api", "--paginate", endpoint]
raw = gh(*cmd)
# gh --paginate concatenates JSON arrays, so we may get multiple arrays
issues = []
for line in raw.strip().splitlines():
line = line.strip()
if not line:
continue
parsed = json.loads(line)
if isinstance(parsed, list):
issues.extend(parsed)
else:
issues.append(parsed)
# Filter out pull requests (they also appear in the issues endpoint)
return [i for i in issues if "pull_request" not in i]
def close_as_duplicate(
issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
) -> None:
"""Close an issue as duplicate of another, adding a comment and label."""
repo_args = ["--repo", repo] if repo else []
if dry_run:
print(f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
return
# Add comment
comment_body = (
f"Closing as duplicate of #{duplicate_of}.\n\n"
"If you believe this is not a duplicate, please reopen and add context "
"explaining how this differs."
)
gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
# Add label
gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
# Close with not_planned reason
gh(
"api",
f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
"-X",
"PATCH",
"-f",
"state=closed",
"-f",
"state_reason=not_planned",
)
print(f" Closed #{issue_number} as duplicate of #{duplicate_of}")
def find_duplicate(
issue: dict, candidates: list[dict], threshold: float
) -> dict | None:
"""Return the first candidate whose normalized title is above threshold."""
norm = normalize_title(issue["title"])
for candidate in candidates:
if candidate["number"] == issue["number"]:
continue
cand_norm = normalize_title(candidate["title"])
ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
if ratio >= threshold:
return candidate
return None
def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
"""Compare every issue against all older issues. Returns count of duplicates found."""
# Sort oldest first
issues.sort(key=lambda i: i["number"])
closed_count = 0
for idx, issue in enumerate(issues):
older = issues[:idx]
if not older:
continue
dup = find_duplicate(issue, older, threshold)
if dup:
ratio = difflib.SequenceMatcher(
None,
normalize_title(issue["title"]),
normalize_title(dup["title"]),
).ratio()
print(
f"#{issue['number']}: \"{issue['title']}\"\n"
f" -> duplicate of #{dup['number']}: \"{dup['title']}\" "
f"({ratio:.0%} similar)"
)
close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
closed_count += 1
return closed_count
def check_single(
issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
) -> bool:
"""Check a single issue against all older open issues. Returns True if duplicate found."""
target = None
for i in issues:
if i["number"] == issue_number:
target = i
break
if target is None:
print(f"Issue #{issue_number} not found among open issues.")
return False
older = [i for i in issues if i["number"] < issue_number]
dup = find_duplicate(target, older, threshold)
if dup:
ratio = difflib.SequenceMatcher(
None,
normalize_title(target["title"]),
normalize_title(dup["title"]),
).ratio()
print(
f"#{target['number']}: \"{target['title']}\"\n"
f" -> duplicate of #{dup['number']}: \"{dup['title']}\" "
f"({ratio:.0%} similar)"
)
close_as_duplicate(issue_number, dup["number"], repo, dry_run)
return True
print(f"#{issue_number}: no duplicate found above threshold {threshold}")
return False
def main() -> None:
parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--scan", action="store_true", help="Scan all open issues")
mode.add_argument("--issue-number", type=int, help="Check a single issue number")
parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
args = parser.parse_args()
dry_run = not args.close
if dry_run:
print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
print("Fetching open issues...")
issues = fetch_open_issues(args.repo)
print(f"Found {len(issues)} open issues.\n")
if args.scan:
count = scan_all(issues, args.threshold, args.repo, dry_run)
print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
else:
found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
sys.exit(0 if found else 0) # Always exit 0; finding no dup is not an error
if __name__ == "__main__":
main()

1
.github/workflows/auto_update_price_and_context_window.yml vendored
View file

@ -7,6 +7,7 @@ on:
jobs:
auto_update_price_and_context_window:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

68
.github/workflows/check_duplicate_issues.yml vendored
View file

@ -2,47 +2,51 @@ name: Check Duplicate Issues
on:
issues:
types: [opened]
types: [opened, edited]
jobs:
check-duplicates:
if: github.event.action == 'opened'
check-duplicate:
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
contents: read
steps:
- name: Install Claude Code
run: npm install -g @anthropic-ai/claude-code
- name: Check duplicates
env:
ANTHROPIC_API_KEY: ${{ secrets.LITELLM_VIRTUAL_KEY }}
ANTHROPIC_BASE_URL: ${{ secrets.LITELLM_BASE_URL }}
- name: Check for potential duplicates
uses: wow-actions/potential-duplicates@v1
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PROMPT: |
A new issue has been created in the ${{ github.repository }} repository.
label: potential-duplicate
threshold: 0.6
reaction: eyes
comment: |
**⚠️ Potential duplicate detected**
Issue number: ${{ github.event.issue.number }}
This issue appears similar to existing issue(s):
{{#issues}}
- [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
{{/issues}}
Lookup this issue with gh issue view ${{ github.event.issue.number }} --repo ${{ github.repository }}.
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
Search through existing issues (excluding #${{ github.event.issue.number }}) to find potential duplicates.
- name: Checkout close script
if: github.event.action == 'opened'
uses: actions/checkout@v4
with:
sparse-checkout: .github/scripts
Use gh issue list --repo ${{ github.repository }} with relevant search terms from the new issue's title and description. Try multiple keyword combinations to search broadly. Check both open and recently closed issues.
- name: Set up Python
if: github.event.action == 'opened'
uses: actions/setup-python@v5
with:
python-version: "3.11"
Consider:
1. Similar titles or descriptions
2. Same error messages or symptoms
3. Related functionality or components
4. Similar feature requests
If you find potential duplicates, post a SINGLE comment on issue #${{ github.event.issue.number }} using gh issue comment ${{ github.event.issue.number }} --repo ${{ github.repository }} with this format:
_This comment was generated by an LLM and may be inaccurate._
This issue might be a duplicate of existing issues. Please check:
- #[issue_number]: [brief description of similarity]
If you find NO duplicates, do NOT post any comment. Stay silent.
run: claude -p "$PROMPT" --model sonnet --max-turns 10 --allowedTools "Bash(gh issue *)"
- name: Auto-close if high-confidence duplicate
if: github.event.action == 'opened'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
python3 .github/scripts/close_duplicate_issues.py \
--issue-number ${{ github.event.issue.number }} \
--repo ${{ github.repository }} \
--threshold 0.85 \
--close

52
.github/workflows/check_duplicate_prs.yml vendored
View file

@ -1,52 +0,0 @@
name: Check Duplicate PRs
on:
pull_request_target:
types: [opened]
jobs:
check-duplicates:
if: |
github.event.pull_request.user.login != 'ishaan-jaff' &&
github.event.pull_request.user.login != 'krrishdholakia' &&
github.event.pull_request.user.login != 'actions-user' &&
!endsWith(github.event.pull_request.user.login, '[bot]')
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- name: Install Claude Code
run: npm install -g @anthropic-ai/claude-code
- name: Check duplicates
env:
ANTHROPIC_API_KEY: ${{ secrets.LITELLM_VIRTUAL_KEY }}
ANTHROPIC_BASE_URL: ${{ secrets.LITELLM_BASE_URL }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PROMPT: |
A new PR has been opened in the ${{ github.repository }} repository.
PR number: ${{ github.event.pull_request.number }}
Lookup this PR with gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }}.
Search through existing open PRs (excluding #${{ github.event.pull_request.number }}) to find potential duplicates.
Use gh pr list --repo ${{ github.repository }} with relevant search terms from the new PR's title and description. Try multiple keyword combinations to search broadly. Check both open and recently closed PRs.
Consider:
1. Similar titles or descriptions
2. Same bug fix or feature being implemented
3. Related functionality or components
4. Overlapping code changes (same files or areas)
If you find potential duplicates, post a SINGLE comment on PR #${{ github.event.pull_request.number }} using gh pr comment ${{ github.event.pull_request.number }} --repo ${{ github.repository }} with this format:
_This comment was generated by an LLM and may be inaccurate._
This PR might be a duplicate of existing PRs. Please check:
- #[pr_number]: [brief description of similarity]
If you find NO duplicates, do NOT post any comment. Stay silent.
run: claude -p "$PROMPT" --model sonnet --max-turns 10 --allowedTools "Bash(gh pr *)"

52
.github/workflows/codeql.yml vendored Normal file
View file

@ -0,0 +1,52 @@
name: "CodeQL"
on:
push:
branches: [main]
pull_request:
branches: [main]
schedule:
# Run weekly on Sundays at 04:00 UTC
- cron: "0 4 * * 0"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
name: Analyze (${{ matrix.language }})
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
security-events: write
packages: read
actions: read
contents: read
strategy:
fail-fast: false
matrix:
include:
- language: actions
build-mode: none
- language: javascript-typescript
build-mode: none
- language: python
build-mode: none
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
config-file: ./.github/codeql/codeql-config.yml
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{ matrix.language }}"

36
.github/workflows/create_daily_staging_branch.yml vendored
View file

@ -41,3 +41,39 @@ jobs:
git push origin $BRANCH_NAME
echo "Successfully created and pushed branch: $BRANCH_NAME"
fi
create-internal-dev-branch:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Create internal dev branch
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Configure Git user
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
# Generate branch name with MM_DD_YYYY format
BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
echo "Creating branch: $BRANCH_NAME"
# Fetch all branches
git fetch --all
# Check if the branch already exists
if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
echo "Branch $BRANCH_NAME already exists. Skipping creation."
else
echo "Creating new branch: $BRANCH_NAME"
# Create the new branch from main
git checkout -b $BRANCH_NAME origin/main
# Push the new branch
git push origin $BRANCH_NAME
echo "Successfully created and pushed branch: $BRANCH_NAME"
fi

9
.github/workflows/ghcr_deploy.yml vendored
View file

@ -299,6 +299,15 @@ jobs:
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
run-observatory-tests:
if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable'
needs: [docker-hub-deploy]
uses: ./.github/workflows/run_observatory_tests.yml
with:
tag: ${{ github.event.inputs.tag }}
commit_hash: ${{ github.event.inputs.commit_hash }}
secrets: inherit
build-and-push-helm-chart:
if: github.event.inputs.release_type != 'dev'
needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]

2
.github/workflows/interpret_load_test.py vendored
View file

@ -123,7 +123,7 @@ if __name__ == "__main__":
+ docker_run_command
+ "\n\n"
+ "### Don't want to maintain your internal proxy? get in touch 🎉"
+ "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
+ "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions"
+ "\n\n"
+ "## Load Test LiteLLM Proxy Results"
+ "\n\n"

94
.github/workflows/publish_enterprise.yml vendored Normal file
View file

@ -0,0 +1,94 @@
name: Publish litellm-enterprise to PyPI
on:
workflow_dispatch:
inputs:
bump:
description: "Version bump type"
required: true
default: "patch"
type: choice
options:
- patch
- minor
- major
jobs:
publish:
runs-on: ubuntu-latest
if: github.repository == 'BerriAI/litellm'
permissions:
contents: write
pull-requests: write
defaults:
run:
working-directory: enterprise
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install Poetry
run: pip install poetry
- name: Bump version
id: bump
run: |
OLD=$(poetry version -s)
poetry version ${{ github.event.inputs.bump }}
NEW=$(poetry version -s)
echo "old=$OLD" >> $GITHUB_OUTPUT
echo "new=$NEW" >> $GITHUB_OUTPUT
- name: Update version refs in root pyproject.toml and requirements.txt
run: |
OLD=${{ steps.bump.outputs.old }}
NEW=${{ steps.bump.outputs.new }}
sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml
sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt
- name: Update poetry.lock
working-directory: .
run: poetry lock
- name: Build
run: poetry build
- name: Commit version bump and create PR
id: create-pr
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
cd ..
BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}"
git checkout -b "$BRANCH"
git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
git push origin "$BRANCH" --force
gh pr create \
--title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \
--body "Version bump for litellm-enterprise. Merge to update main." \
--head "$BRANCH" \
--base main \
|| true
PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url')
echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ github.token }}
- name: Enable auto-merge
run: |
gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash
env:
GH_TOKEN: ${{ github.token }}
- name: Publish to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }}
run: |
pip install twine
twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}*

74
.github/workflows/publish_proxy_extras.yml vendored Normal file
View file

@ -0,0 +1,74 @@
name: Publish litellm-proxy-extras to PyPI
on:
workflow_dispatch:
inputs:
bump:
description: "Version bump type"
required: true
default: "patch"
type: choice
options:
- patch
- minor
- major
jobs:
publish:
runs-on: ubuntu-latest
if: github.repository == 'BerriAI/litellm'
permissions:
contents: write
defaults:
run:
working-directory: litellm-proxy-extras
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install Poetry
run: pip install poetry
- name: Bump version
id: bump
run: |
OLD=$(poetry version -s)
poetry version ${{ github.event.inputs.bump }}
NEW=$(poetry version -s)
echo "old=$OLD" >> $GITHUB_OUTPUT
echo "new=$NEW" >> $GITHUB_OUTPUT
- name: Update version refs in root pyproject.toml and requirements.txt
run: |
OLD=${{ steps.bump.outputs.old }}
NEW=${{ steps.bump.outputs.new }}
sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml
sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt
- name: Update poetry.lock
working-directory: .
run: poetry lock
- name: Build
run: poetry build
- name: Commit version bump
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
cd ..
git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock
git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
git push
- name: Publish to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
run: |
pip install twine
twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}*

225
.github/workflows/run_observatory_tests.yml vendored Normal file
View file

@ -0,0 +1,225 @@
name: Run Observatory Tests
on:
workflow_dispatch:
inputs:
tag:
description: "Docker image tag to test (e.g. v1.61.0.rc1)"
required: true
type: string
commit_hash:
description: "Commit hash (defaults to HEAD of current branch)"
required: false
type: string
workflow_call:
inputs:
tag:
description: "Docker image tag to test"
required: true
type: string
commit_hash:
description: "Commit hash of the release"
required: true
type: string
permissions:
contents: read
env:
LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
jobs:
observatory-tests:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Validate tag input
env:
TAG: ${{ inputs.tag }}
run: |
if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
exit 1
fi
- name: Start LiteLLM container
env:
TAG: ${{ inputs.tag }}
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
run: |
docker run -d \
--name litellm-rc \
-p 4000:4000 \
-v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
-e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
-e AZURE_API_KEY="${AZURE_API_KEY}" \
-e AZURE_API_BASE="${AZURE_API_BASE}" \
"litellm/litellm:${TAG}" \
--config /app/config.yaml --port 4000
- name: Wait for LiteLLM health check
run: |
echo "Waiting for LiteLLM to be ready..."
for i in $(seq 1 30); do
if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
echo "LiteLLM is healthy"
exit 0
fi
echo "Attempt $i/30 - not ready yet, waiting 10s..."
sleep 10
done
echo "LiteLLM failed to start within 5 minutes"
docker logs litellm-rc
exit 1
- name: Start cloudflared tunnel
run: |
# Install cloudflared
curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
chmod +x /usr/local/bin/cloudflared
# Start a quick tunnel (no account needed) and capture the URL
cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
CLOUDFLARED_PID=$!
echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
# Wait for tunnel URL to appear in logs
echo "Waiting for tunnel URL..."
for i in $(seq 1 30); do
TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
if [ -n "$TUNNEL_URL" ]; then
echo "Tunnel URL: $TUNNEL_URL"
echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
exit 0
fi
sleep 2
done
echo "Failed to get tunnel URL"
cat /tmp/cloudflared.log
exit 1
- name: Verify tunnel connectivity
run: |
echo "Testing tunnel at ${{ env.TUNNEL_URL }}..."
# Quick tunnels need time for DNS propagation; retry to avoid
# transient NXDOMAIN (curl exit code 6) on first attempt.
for i in $(seq 1 10); do
if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then
echo "Tunnel is working (attempt $i)"
exit 0
fi
echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
sleep 5
done
echo "Tunnel failed to become reachable after 50s"
cat /tmp/cloudflared.log
exit 1
- name: Trigger observatory test run
id: trigger
env:
OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
run: |
PAYLOAD=$(jq -n \
--arg url "${TUNNEL_URL}" \
--arg key "${LITELLM_MASTER_KEY}" \
'{
deployment_url: $url,
api_key: $key,
test_suite: "TestOAIAzureRelease",
models: ["gpt-4o-mini", "gpt-4o"]
}')
RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
-H "Content-Type: application/json" \
-H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
-d "$PAYLOAD")
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | head -n -1)
echo "Response ($HTTP_CODE): $BODY"
if [ "$HTTP_CODE" -ge 400 ]; then
echo "Failed to trigger test run"
exit 1
fi
# Extract request_id for polling this specific run
REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
echo "Failed to extract request_id from response"
exit 1
fi
echo "Request ID: $REQUEST_ID"
echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
- name: Poll for test completion
id: poll
env:
OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
run: |
TIMEOUT=900 # 15 minutes
INTERVAL=30
ELAPSED=0
while [ $ELAPSED -lt $TIMEOUT ]; do
STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
-H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
echo "Test finished with status: $RUN_STATUS"
echo "$STATUS" > /tmp/observatory_result.json
exit 0
fi
sleep $INTERVAL
ELAPSED=$((ELAPSED + INTERVAL))
done
echo "Timed out waiting for test to complete after ${TIMEOUT}s"
exit 1
- name: Verify test results
run: |
RESULT=$(cat /tmp/observatory_result.json)
echo "Full result: $RESULT"
STATUS=$(echo "$RESULT" | jq -r '.status')
TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
ERROR=$(echo "$RESULT" | jq -r '.error // empty')
echo "Status: $STATUS"
echo "Test passed: $TEST_PASSED"
echo "Failure rate: $FAILURE_RATE"
if [ -n "$ERROR" ]; then
echo "Error: $ERROR"
fi
if [ "$STATUS" = "failed" ]; then
echo "Test run failed"
exit 1
fi
if [ "$TEST_PASSED" != "true" ]; then
echo "Tests did not pass (failure rate: $FAILURE_RATE)"
exit 1
fi
echo "All tests passed!"
- name: Print LiteLLM logs on failure
if: failure()
run: |
docker logs litellm-rc 2>/dev/null || true
cat /tmp/cloudflared.log 2>/dev/null || true
- name: Cleanup
if: always()
run: |
kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true
docker rm -f litellm-rc 2>/dev/null || true

47
.github/workflows/scan_duplicate_issues.yml vendored Normal file
View file

@ -0,0 +1,47 @@
name: Scan Duplicate Issues (One-Time)
on:
workflow_dispatch:
inputs:
threshold:
description: "Similarity threshold (0-1)"
required: false
default: "0.85"
close:
description: "Actually close duplicates (false = dry run)"
required: false
type: boolean
default: false
jobs:
scan:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Checkout scripts
uses: actions/checkout@v4
with:
sparse-checkout: .github/scripts
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Scan for duplicate issues
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
INPUT_THRESHOLD: ${{ inputs.threshold }}
INPUT_CLOSE: ${{ inputs.close }}
run: |
CLOSE_FLAG=""
if [ "$INPUT_CLOSE" = "true" ]; then
CLOSE_FLAG="--close"
fi
python3 .github/scripts/close_duplicate_issues.py \
--scan \
--repo ${{ github.repository }} \
--threshold "$INPUT_THRESHOLD" \
$CLOSE_FLAG

33
.github/workflows/test-linting.yml vendored
View file

@ -32,7 +32,6 @@ jobs:
run: |
poetry lock
poetry install --with dev
poetry run pip install openai==1.100.1
- name: Run Black formatting
run: |
@ -74,3 +73,35 @@ jobs:
- name: Check import safety
run: |
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
secret-scan:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Run secret scan test
run: |
pip install pytest
pytest tests/litellm/test_no_hardcoded_secrets.py -v
- name: Run ggshield secret scan
env:
GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
run: |
if [ -n "$GITGUARDIAN_API_KEY" ]; then
pip install ggshield
ggshield secret scan repo .
else
echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
fi

2
.github/workflows/test-litellm.yml vendored
View file

@ -38,7 +38,7 @@ jobs:
poetry run pip install "google-genai==1.22.0"
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.22"
poetry run pip install "python-multipart>=0.0.20"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |

90
.github/workflows/test-proxy-e2e-azure-batches.yml vendored Normal file
View file

@ -0,0 +1,90 @@
name: Proxy E2E Azure Batches Tests
on:
pull_request:
branches: [main]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
proxy_e2e_azure_batches_tests:
runs-on: ubuntu-latest
timeout-minutes: 30
services:
postgres:
image: postgres:15
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Cache Poetry dependencies
uses: actions/cache@v4
with:
path: |
~/.cache/pypoetry
~/.cache/pip
.venv
key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }}
restore-keys: |
${{ runner.os }}-poetry-e2e-batches-
${{ runner.os }}-poetry-
- name: Install dependencies
run: |
poetry config virtualenvs.in-project true
poetry install --with dev,proxy-dev --extras "proxy"
poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
- name: Setup litellm-enterprise
run: |
poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Generate Prisma client
run: |
poetry run prisma generate --schema litellm/proxy/schema.prisma
- name: Run Prisma migrations
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
run: |
cd litellm/proxy
poetry run prisma migrate deploy --schema schema.prisma
cd ../..
- name: Run Azure Batch E2E Tests
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
USE_LOCAL_LITELLM: "true"
USE_MOCK_MODELS: "true"
USE_STATE_TRACKER: "true"
LITELLM_LOG: DEBUG
run: |
poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
-vv -s -k "test_e2e_managed_batch" \
--tb=short \
--maxfail=3 \
--durations=10

1
.gitignore vendored
View file

@ -89,6 +89,7 @@ tests/test_custom_dir/*
test.py
litellm_config.yaml
!.github/observatory/litellm_config.yaml
.cursor
.vscode/launch.json
litellm/proxy/to_delete_loadtest_work/*

86
AGENTS.md
View file

@ -109,6 +109,8 @@ Key files:
- `litellm/proxy/auth/` - Authentication logic
- `litellm/proxy/management_endpoints/` - Admin API endpoints
**Database (proxy)**: Use Prisma model methods (`prisma_client.db.<model>.upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details.
## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
LiteLLM supports MCP for agent workflows:
@ -174,6 +176,43 @@ When opening issues or pull requests, follow these templates:
3. **Rate Limits**: Respect provider rate limits in tests
4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
5. **Dependencies**: Keep dependencies minimal and well-justified
6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift)
8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.
**Example of BAD** (hardcoded model checks):
```python
@staticmethod
def _is_effort_supported_model(model: str) -> bool:
"""Check if the model supports the output_config.effort parameter..."""
model_lower = model.lower()
if AnthropicConfig._is_claude_4_6_model(model):
return True
return any(
v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5")
)
```
**Example of GOOD** (config-driven or helper that reads from config):
```python
if (
"claude-3-7-sonnet" in model
or AnthropicConfig._is_claude_4_6_model(model)
or supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
)
):
...
```
Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history.
## HELPFUL RESOURCES
@ -187,4 +226,49 @@ When opening issues or pull requests, follow these templates:
- Check similar provider implementations
- Ensure comprehensive test coverage
- Update documentation appropriately
- Consider backward compatibility impact
- Consider backward compatibility impact
## Cursor Cloud specific instructions
### Environment
- Poetry is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
- Python 3.12, Node 22 are pre-installed.
- The virtual environment lives under `~/.cache/pypoetry/virtualenvs/`.
### Running the proxy server
Start the proxy with a config file:
```bash
poetry run litellm --config dev_config.yaml --port 4000
```
The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
### Running tests
See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`.
- The `--timeout` pytest flag is NOT available; don't pass it.
- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
- Black `--check` may report pre-existing formatting issues; this does not block test runs.
- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file.
### Lint
```bash
cd litellm && poetry run ruff check .
```
Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
### UI Dashboard development
- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000.
- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
- SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`

46
CLAUDE.md
View file

@ -97,13 +97,57 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- Integration tests for each provider in `tests/llm_translation/`
- Proxy tests in `tests/proxy_unit_tests/`
- Load tests in `tests/load_tests/`
- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
### UI / Backend Consistency
- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
### MCP OAuth / OpenAPI Transport Mapping
- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.
- `client_id` should be optional in the `/authorize` endpoint — if the server has a stored `client_id` in credentials, use that. Never require callers to re-supply it.
### MCP Credential Storage
- OAuth credentials and BYOK credentials share the `litellm_mcpusercredentials` table, distinguished by a `"type"` field in the JSON payload (`"oauth2"` vs plain string).
- When deleting OAuth credentials, check type before deleting to avoid accidentally deleting a BYOK credential for the same `(user_id, server_id)` pair.
- Always pass the raw `expires_at` timestamp to the client — never set it to `None` for expired credentials. Let the frontend compute the "Expired" display state from the timestamp.
- Use `RecordNotFoundError` (not bare `except Exception`) when catching "already deleted" in credential delete endpoints.
### Browser Storage Safety (UI)
- Never write LiteLLM access tokens or API keys to `localStorage` — use `sessionStorage` only. `localStorage` survives browser close and is readable by any injected script (XSS).
- Shared utility functions (e.g. `extractErrorMessage`) belong in `src/utils/` — never define them inline in hooks or duplicate them across files.
### Database Migrations
- Prisma handles schema migrations
- Migration files auto-generated with `prisma migrate dev`
- Always test migrations against both PostgreSQL and SQLite
### Proxy database access
- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`.
- Use the generated client: `prisma_client.db.<model>` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code.
- **No N+1 queries.** Never query the DB inside a loop. Batch-fetch with `{"in": ids}` and distribute in-memory.
- **Batch writes.** Use `create_many`/`update_many`/`delete_many` instead of individual calls (these return counts only; `update_many`/`delete_many` no-op silently on missing rows). When multiple separate writes target the same table (e.g. in `batch_()`), order by primary key to avoid deadlocks.
- **Push work to the DB.** Filter, sort, group, and aggregate in SQL, not Python. Verify Prisma generates the expected SQL — e.g. prefer `group_by` over `find_many(distinct=...)` which does client-side processing.
- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
### Enterprise Features
- Enterprise-specific code in `enterprise/` directory
- Optional features enabled via environment variables
- Separate licensing and authentication for enterprise features
- Separate licensing and authentication for enterprise features
### HTTP Client Cache Safety
- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`.
### Troubleshooting: DB schema out of sync after proxy restart
`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields.
**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue.
**Fix options:**
1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.

31
Dockerfile
View file

@ -49,7 +49,7 @@ USER root
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
# SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
# levels inside its dependency tree. `npm install -g <pkg>` only creates a
# SEPARATE global package, it does NOT replace npm's internal copies.
@ -64,7 +64,21 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
npm cache clean --force
find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
# SECURITY FIX: patch npm's own package.json metadata so scanners see the
# actual installed versions instead of the stale declared dependencies.
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
# Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is
# no longer visible to image scanners. The globally installed npm@latest
# at /usr/local/lib/node_modules/npm/ remains fully functional.
{ apk del --no-cache npm 2>/dev/null || true; }
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -90,14 +104,21 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done
# Install semantic_router and aurelio-sdk using script

4
README.md
View file

@ -203,7 +203,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
{
"mcpServers": {
"LiteLLM": {
"url": "http://localhost:4000/mcp",
"url": "http://localhost:4000/mcp/",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
@ -399,7 +399,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
# Enterprise
For companies that need better security, user management and professional support
[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
This covers:
- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**

3
ci_cd/security_scans.sh
View file

@ -160,6 +160,9 @@ run_grype_scans() {
"CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
"GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
"GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
"CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
"GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
)
# Build JSON array of allowlisted CVE IDs for jq

2
cookbook/benchmark/readme.md
View file

@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?':
```
## Support
**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.

119
cookbook/gollem_go_agent_framework/README.md Normal file
View file

@ -0,0 +1,119 @@
# Gollem Go Agent Framework with LiteLLM
A working example showing how to use [gollem](https://github.com/fugue-labs/gollem), a production-grade Go agent framework, with LiteLLM as a proxy gateway. This lets Go developers access 100+ LLM providers through a single proxy while keeping compile-time type safety for tools and structured output.
## Quick Start
### 1. Start LiteLLM Proxy
```bash
# Simple start with a single model
litellm --model gpt-4o
# Or with the example config for multi-provider access
litellm --config proxy_config.yaml
```
### 2. Run the examples
```bash
# Install Go dependencies
go mod tidy
# Basic agent
go run ./basic
# Agent with type-safe tools
go run ./tools
# Streaming responses
go run ./streaming
```
## Configuration
The included `proxy_config.yaml` sets up three providers through LiteLLM:
```yaml
model_list:
- model_name: gpt-4o # OpenAI
- model_name: claude-sonnet # Anthropic
- model_name: gemini-pro # Google Vertex AI
```
Switch providers in Go by changing a single string — no code changes needed:
```go
model := openai.NewLiteLLM("http://localhost:4000",
openai.WithModel("gpt-4o"), // OpenAI
// openai.WithModel("claude-sonnet"), // Anthropic
// openai.WithModel("gemini-pro"), // Google
)
```
## Examples
### `basic/` — Basic Agent
Connects gollem to LiteLLM and runs a simple prompt. Demonstrates the `NewLiteLLM` constructor and basic agent creation.
### `tools/` — Type-Safe Tools
Shows gollem's compile-time type-safe tool framework working through LiteLLM's tool-use passthrough. The tool parameters are Go structs with JSON tags — the schema is generated automatically at compile time.
### `streaming/` — Streaming Responses
Real-time token streaming using Go 1.23+ range-over-function iterators, proxied through LiteLLM's SSE passthrough.
## How It Works
Gollem's `openai.NewLiteLLM()` constructor creates an OpenAI-compatible provider pointed at your LiteLLM proxy. Since LiteLLM speaks the OpenAI API protocol, everything works out of the box:
- **Chat completions** — standard request/response
- **Tool use** — LiteLLM passes tool definitions and calls through transparently
- **Streaming** — Server-Sent Events proxied through LiteLLM
- **Structured output** — JSON schema response format works with supporting models
```
Go App (gollem) → LiteLLM Proxy → OpenAI / Anthropic / Google / ...
```
## Why Use This?
- **Type-safe Go**: Compile-time type checking for tools, structured output, and agent configuration — no runtime surprises
- **Single proxy, many models**: Switch between OpenAI, Anthropic, Google, and 100+ other providers by changing a model name string
- **Zero-dependency core**: gollem's core has no external dependencies — just stdlib
- **Single binary deployment**: `go build` produces one binary, no pip/venv/Docker needed
- **Cost tracking & rate limiting**: LiteLLM handles cost tracking, rate limits, and fallbacks at the proxy layer
## Environment Variables
```bash
# Required for providers you want to use (set in LiteLLM config or env)
export OPENAI_API_KEY="sk-..."
export ANTHROPIC_API_KEY="sk-ant-..."
# Optional: point to a non-default LiteLLM proxy
export LITELLM_PROXY_URL="http://localhost:4000"
```
## Troubleshooting
**Connection errors?**
- Make sure LiteLLM is running: `litellm --model gpt-4o`
- Check the URL is correct (default: `http://localhost:4000`)
**Model not found?**
- Verify the model name matches what's configured in LiteLLM
- Run `curl http://localhost:4000/models` to see available models
**Tool calls not working?**
- Ensure the underlying model supports tool use (GPT-4o, Claude, Gemini)
- Check LiteLLM logs for any provider-specific errors
## Learn More
- [gollem GitHub](https://github.com/fugue-labs/gollem)
- [gollem API Reference](https://pkg.go.dev/github.com/fugue-labs/gollem/core)
- [LiteLLM Proxy Docs](https://docs.litellm.ai/docs/simple_proxy)
- [LiteLLM Supported Models](https://docs.litellm.ai/docs/providers)

41
cookbook/gollem_go_agent_framework/basic/main.go Normal file
View file

@ -0,0 +1,41 @@
// Basic gollem agent connected to a LiteLLM proxy.
//
// Usage:
//
// litellm --model gpt-4o # start proxy in another terminal
// go run ./basic
package main
import (
"context"
"fmt"
"log"
"os"
"github.com/fugue-labs/gollem/core"
"github.com/fugue-labs/gollem/provider/openai"
)
func main() {
proxyURL := "http://localhost:4000"
if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
proxyURL = u
}
// Connect to LiteLLM proxy. NewLiteLLM creates an OpenAI-compatible
// provider pointed at the given URL.
model := openai.NewLiteLLM(proxyURL,
openai.WithModel("gpt-4o"), // any model name configured in LiteLLM
)
// Create and run a simple agent.
agent := core.NewAgent[string](model,
core.WithSystemPrompt[string]("You are a helpful assistant. Be concise."),
)
result, err := agent.Run(context.Background(), "Explain quantum computing in two sentences.")
if err != nil {
log.Fatal(err)
}
fmt.Println(result.Output)
}

5
cookbook/gollem_go_agent_framework/go.mod Normal file
View file

@ -0,0 +1,5 @@
module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework
go 1.25.1
require github.com/fugue-labs/gollem v0.1.0

2
cookbook/gollem_go_agent_framework/go.sum Normal file
View file

@ -0,0 +1,2 @@
github.com/fugue-labs/gollem v0.1.0 h1:QexYnvkb44QZFEljgAePqMIGZjgsbk0Y5GJ2jYYgfa8=
github.com/fugue-labs/gollem v0.1.0/go.mod h1:htW1YO81uysSKVOkYJtxhGCFrzm+36HBFxEWuECoHKQ=

16
cookbook/gollem_go_agent_framework/proxy_config.yaml Normal file
View file

@ -0,0 +1,16 @@
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: gemini-pro
litellm_params:
model: vertex_ai/gemini-2.0-flash
vertex_project: my-project
vertex_location: us-central1

56
cookbook/gollem_go_agent_framework/streaming/main.go Normal file
View file

@ -0,0 +1,56 @@
// Streaming responses from gollem through LiteLLM.
//
// Uses Go 1.23+ range-over-function iterators for real-time token
// streaming via LiteLLM's SSE passthrough.
//
// Usage:
//
// litellm --model gpt-4o
// go run ./streaming
package main
import (
"context"
"fmt"
"log"
"os"
"github.com/fugue-labs/gollem/core"
"github.com/fugue-labs/gollem/provider/openai"
)
func main() {
proxyURL := "http://localhost:4000"
if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
proxyURL = u
}
model := openai.NewLiteLLM(proxyURL,
openai.WithModel("gpt-4o"),
)
agent := core.NewAgent[string](model)
// RunStream returns a streaming result that yields tokens as they arrive.
stream, err := agent.RunStream(context.Background(), "Write a haiku about distributed systems")
if err != nil {
log.Fatal(err)
}
// StreamText yields text chunks in real-time.
// The boolean argument controls whether deltas (true) or accumulated
// text (false) is returned.
fmt.Print("Response: ")
for text, err := range stream.StreamText(true) {
if err != nil {
log.Fatal(err)
}
fmt.Print(text)
}
fmt.Println()
// After streaming completes, the final response is available.
resp := stream.Response()
fmt.Printf("\nTokens used: input=%d, output=%d\n",
resp.Usage.InputTokens, resp.Usage.OutputTokens)
}

64
cookbook/gollem_go_agent_framework/tools/main.go Normal file
View file

@ -0,0 +1,64 @@
// Gollem agent with type-safe tools through LiteLLM.
//
// The tool parameters are Go structs — gollem generates the JSON schema
// automatically at compile time. LiteLLM passes tool definitions through
// transparently to the underlying provider.
//
// Usage:
//
// litellm --model gpt-4o
// go run ./tools
package main
import (
"context"
"fmt"
"log"
"os"
"github.com/fugue-labs/gollem/core"
"github.com/fugue-labs/gollem/provider/openai"
)
// WeatherParams defines the tool's input schema via struct tags.
// The JSON schema is generated at compile time — no runtime reflection needed.
type WeatherParams struct {
City string `json:"city" description:"City name to get weather for"`
Unit string `json:"unit,omitempty" description:"Temperature unit: celsius or fahrenheit"`
}
func main() {
proxyURL := "http://localhost:4000"
if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
proxyURL = u
}
model := openai.NewLiteLLM(proxyURL,
openai.WithModel("gpt-4o"),
)
// Define a type-safe tool. The function signature enforces correct types.
weatherTool := core.FuncTool[WeatherParams](
"get_weather",
"Get current weather for a city",
func(ctx context.Context, p WeatherParams) (string, error) {
unit := p.Unit
if unit == "" {
unit = "fahrenheit"
}
// In production, call a real weather API here.
return fmt.Sprintf("Weather in %s: 72°F (22°C), sunny", p.City), nil
},
)
agent := core.NewAgent[string](model,
core.WithTools[string](weatherTool),
core.WithSystemPrompt[string]("You are a helpful weather assistant. Use the get_weather tool to answer weather questions."),
)
result, err := agent.Run(context.Background(), "What's the weather like in San Francisco and Tokyo?")
if err != nil {
log.Fatal(err)
}
fmt.Println(result.Output)
}

4
deploy/charts/litellm-helm/README.md
View file

@ -36,6 +36,10 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` |
| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` |
| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` |
| `livenessProbe.*` | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` |
| `readinessProbe.*` | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` |
| `startupProbe.*` | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` |
| `resources.*` | CPU/memory requests and limits for the LiteLLM container. | `{}` |
| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` |
| `ingress.labels` | Additional labels for the Ingress resource | `{}` |
| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A |

14
deploy/charts/litellm-helm/templates/_helpers.tpl
View file

@ -61,6 +61,20 @@ Create the name of the service account to use
{{- end }}
{{- end }}
{{/*
Create the service account name used by migration jobs.
When Helm hooks are enabled, pre-install/pre-upgrade hooks run before normal resources.
If this chart is creating the ServiceAccount, it is not yet available for the hook job,
so fall back to "default" (or an explicit override) to avoid a cyclic dependency.
*/}}
{{- define "litellm.migrationServiceAccountName" -}}
{{- if and .Values.migrationJob.hooks.helm.enabled .Values.serviceAccount.create }}
{{- default "default" .Values.migrationJob.serviceAccountName }}
{{- else }}
{{- include "litellm.serviceAccountName" . }}
{{- end }}
{{- end }}
{{/*
Get redis service name
*/}}

2
deploy/charts/litellm-helm/templates/configmap-litellm.yaml
View file

@ -6,4 +6,4 @@ metadata:
data:
config.yaml: |
{{ .Values.proxy_config | toYaml | indent 6 }}
{{- end }}
{{- end }}

29
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -13,6 +13,10 @@ spec:
{{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
replicas: {{ .Values.replicaCount }}
{{- end }}
{{- with .Values.strategy }}
strategy:
{{- toYaml . | nindent 4 }}
{{- end }}
selector:
matchLabels:
{{- include "litellm.selectorLabels" . | nindent 6 }}
@ -158,18 +162,31 @@ spec:
{{- end }}
livenessProbe:
httpGet:
path: /health/liveliness
path: {{ .Values.livenessProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
successThreshold: {{ .Values.livenessProbe.successThreshold }}
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
readinessProbe:
httpGet:
path: /health/readiness
path: {{ .Values.readinessProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
successThreshold: {{ .Values.readinessProbe.successThreshold }}
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
startupProbe:
httpGet:
path: /health/readiness
path: {{ .Values.startupProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
failureThreshold: 30
periodSeconds: 10
initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.startupProbe.periodSeconds }}
timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
successThreshold: {{ .Values.startupProbe.successThreshold }}
failureThreshold: {{ .Values.startupProbe.failureThreshold }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
@ -235,4 +252,4 @@ spec:
{{- if .Values.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
{{- end }}
{{- end }}

2
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -34,7 +34,7 @@ spec:
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
{{- with .Values.migrationJob.extraInitContainers }}
initContainers:
{{- toYaml . | nindent 8 }}

148
deploy/charts/litellm-helm/tests/deployment_tests.yaml
View file

@ -159,4 +159,150 @@ tests:
value: -c
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
value: echo "Container stopping"
value: echo "Container stopping"
- it: should render background health check settings from proxy_config.general_settings
template: configmap-litellm.yaml
set:
proxy_config.general_settings.background_health_checks: true
proxy_config.general_settings.health_check_interval: 240
proxy_config.general_settings.health_check_concurrency: 16
proxy_config.general_settings.health_check_details: false
asserts:
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*background_health_checks:\s*true$'
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*health_check_interval:\s*240$'
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*health_check_concurrency:\s*16$'
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*health_check_details:\s*false$'
- it: should allow overriding liveness, readiness, and startup probes
template: deployment.yaml
set:
livenessProbe:
path: /custom/livez
initialDelaySeconds: 5
periodSeconds: 15
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
path: /custom/readyz
initialDelaySeconds: 10
periodSeconds: 20
timeoutSeconds: 6
successThreshold: 1
failureThreshold: 6
startupProbe:
path: /custom/startupz
initialDelaySeconds: 15
periodSeconds: 25
timeoutSeconds: 7
successThreshold: 1
failureThreshold: 40
asserts:
- equal:
path: spec.template.spec.containers[0].livenessProbe.httpGet.path
value: /custom/livez
- equal:
path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
value: 5
- equal:
path: spec.template.spec.containers[0].readinessProbe.httpGet.path
value: /custom/readyz
- equal:
path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
value: 6
- equal:
path: spec.template.spec.containers[0].startupProbe.httpGet.path
value: /custom/startupz
- equal:
path: spec.template.spec.containers[0].startupProbe.failureThreshold
value: 40
- it: should render container resources from values
template: deployment.yaml
set:
resources:
limits:
cpu: 500m
memory: 2Gi
requests:
cpu: 250m
memory: 1Gi
asserts:
- equal:
path: spec.template.spec.containers[0].resources.limits.cpu
value: 500m
- equal:
path: spec.template.spec.containers[0].resources.limits.memory
value: 2Gi
- equal:
path: spec.template.spec.containers[0].resources.requests.cpu
value: 250m
- equal:
path: spec.template.spec.containers[0].resources.requests.memory
value: 1Gi
- it: should keep default probes and empty resources unchanged
template: deployment.yaml
asserts:
- equal:
path: spec.template.spec.containers[0].livenessProbe.httpGet.path
value: /health/liveliness
- equal:
path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds
value: 0
- equal:
path: spec.template.spec.containers[0].livenessProbe.periodSeconds
value: 10
- equal:
path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
value: 1
- equal:
path: spec.template.spec.containers[0].livenessProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].livenessProbe.failureThreshold
value: 3
- equal:
path: spec.template.spec.containers[0].readinessProbe.httpGet.path
value: /health/readiness
- equal:
path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds
value: 0
- equal:
path: spec.template.spec.containers[0].readinessProbe.periodSeconds
value: 10
- equal:
path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
value: 1
- equal:
path: spec.template.spec.containers[0].readinessProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].readinessProbe.failureThreshold
value: 3
- equal:
path: spec.template.spec.containers[0].startupProbe.httpGet.path
value: /health/readiness
- equal:
path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds
value: 0
- equal:
path: spec.template.spec.containers[0].startupProbe.periodSeconds
value: 10
- equal:
path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
value: 1
- equal:
path: spec.template.spec.containers[0].startupProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].startupProbe.failureThreshold
value: 30
- equal:
path: spec.template.spec.containers[0].resources
value: {}

65
deploy/charts/litellm-helm/tests/migrations-job_tests.yaml
View file

@ -124,4 +124,67 @@ tests:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL
name: DATABASE_URL
- it: should use default service account for helm hooks when serviceAccount.create is true
template: migrations-job.yaml
set:
migrationJob:
enabled: true
hooks:
helm:
enabled: true
serviceAccount:
create: true
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: default
- it: should use migrationJob.serviceAccountName override for helm hooks when serviceAccount.create is true
template: migrations-job.yaml
set:
migrationJob:
enabled: true
serviceAccountName: migration-sa
hooks:
helm:
enabled: true
serviceAccount:
create: true
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: migration-sa
- it: should use chart service account when helm hooks are disabled
template: migrations-job.yaml
set:
migrationJob:
enabled: true
hooks:
helm:
enabled: false
serviceAccount:
create: true
name: my-custom-sa
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: my-custom-sa
- it: should use pre-existing service account when helm hooks are enabled but serviceAccount.create is false
template: migrations-job.yaml
set:
migrationJob:
enabled: true
hooks:
helm:
enabled: true
serviceAccount:
create: false
name: pre-existing-sa
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: pre-existing-sa

37
deploy/charts/litellm-helm/values.yaml
View file

@ -35,6 +35,14 @@ deploymentLabels: {}
podAnnotations: {}
podLabels: {}
# -- Deployment strategy configuration
# Example:
# type: RollingUpdate
# rollingUpdate:
# maxUnavailable: 0
# maxSurge: 1
strategy: {}
terminationGracePeriodSeconds: 90
topologySpreadConstraints:
[]
@ -84,6 +92,31 @@ service:
separateHealthApp: false
separateHealthPort: 8081
# Probe tuning for proxy container
livenessProbe:
path: /health/liveliness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
readinessProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
startupProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 30
ingress:
enabled: false
className: "nginx"
@ -274,6 +307,10 @@ migrationJob:
retries: 3 # Number of retries for the Job in case of failure
backoffLimit: 4 # Backoff limit for Job restarts
disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
# Optional service account for the migration job.
# Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true.
# In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default".
serviceAccountName: ""
annotations: {}
ttlSecondsAfterFinished: 120
resources: {}

13
dev_config.yaml Normal file
View file

@ -0,0 +1,13 @@
model_list:
- model_name: fake-openai-endpoint
litellm_params:
model: openai/fake-model
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
general_settings:
master_key: sk-1234
litellm_settings:
drop_params: True
telemetry: False

28
docker/Dockerfile.custom_ui
View file

@ -5,8 +5,21 @@ FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
WORKDIR /app
# Install Node.js and npm (adjust version as needed)
RUN apt-get update && apt-get install -y nodejs npm && \
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
RUN apt-get update && apt-get upgrade -y \
libxml2 \
libexpat1 \
openssl \
libssl3 \
git \
libkrb5-3 \
libglib2.0-0 \
wget \
libaom3 \
libxslt1.1 \
libgnutls30 \
libc6 && \
apt-get install -y nodejs npm && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -17,7 +30,16 @@ RUN apt-get update && apt-get install -y nodejs npm && \
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
npm cache clean --force
find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
apt-get purge -y npm
# Copy the UI source into the container
COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard

26
docker/Dockerfile.database
View file

@ -50,7 +50,7 @@ USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -61,7 +61,16 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
npm cache clean --force
find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
{ apk del --no-cache npm 2>/dev/null || true; }
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -79,14 +88,21 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done
# Install semantic_router and aurelio-sdk using script

43
docker/Dockerfile.dev
View file

@ -56,13 +56,26 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# Install only runtime dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
libssl3 \
RUN apt-get update && apt-get upgrade -y \
libxml2 \
libexpat1 \
openssl \
libssl3 \
git \
libkrb5-3 \
libglib2.0-0 \
wget \
libaom3 \
libxslt1.1 \
libgnutls30 \
libc6 \
&& apt-get install -y --no-install-recommends \
libssl3 \
libatomic1 \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
&& npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -73,7 +86,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done \
&& npm cache clean --force
&& find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done \
&& find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
&& npm cache clean --force \
&& apt-get purge -y npm
WORKDIR /app
@ -95,14 +117,21 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done
# Generate prisma client and set permissions

29
docker/Dockerfile.non_root
View file

@ -80,7 +80,7 @@ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
XDG_CACHE_HOME=/app/.cache \
PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.12.0 \
RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.13.1 \
&& mkdir -p /app/.cache/npm
RUN NPM_CONFIG_CACHE=/app/.cache/npm \
@ -105,7 +105,8 @@ RUN for i in 1 2 3; do \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done \
&& npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
&& apk upgrade --no-cache nodejs \
&& npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -116,7 +117,16 @@ RUN for i in 1 2 3; do \
&& find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done \
&& npm cache clean --force
&& find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done \
&& find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
&& npm cache clean --force \
&& { apk del --no-cache npm 2>/dev/null || true; }
# Copy artifacts from builder
COPY --from=builder /app/requirements.txt /app/requirements.txt
@ -162,14 +172,21 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done
# Permissions, cleanup, and Prisma prep

147
docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md Normal file
View file

@ -0,0 +1,147 @@
---
slug: anthropic-wildcard-model-access-incident
title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload"
date: 2026-02-23T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
tags: [incident-report, proxy, auth, model-access]
hide_table_of_contents: false
---
**Date:** Feb 23, 2026
**Duration:** ~3 hours
**Severity:** High (for users with provider wildcard access rules)
**Status:** Resolved
## Summary
When a new Anthropic model (e.g. `claude-sonnet-4-6`) was added to the LiteLLM model cost map and a cost map reload was triggered, requests to the new model were rejected with:
```
key not allowed to access model. This key can only access models=['anthropic/*']. Tried to access claude-sonnet-4-6.
```
The reload updated `litellm.model_cost` correctly but never re-ran `add_known_models()`, so `litellm.anthropic_models` (the in-memory set used by the wildcard resolver) remained stale. The new model was invisible to the `anthropic/*` wildcard even though the cost map knew about it.
- **LLM calls:** All requests to newly-added Anthropic models were blocked with a 401.
- **Existing models:** Unaffected — only models missing from the stale provider set were impacted.
- **Other providers:** Same bug class existed for any provider wildcard (e.g. `openai/*`, `gemini/*`).
{/* truncate */}
---
## Background
LiteLLM supports provider-level wildcard access rules. When an admin configures a key or team with `models=['anthropic/*']`, any model whose provider resolves to `anthropic` should be allowed. The resolution happens in `_model_custom_llm_provider_matches_wildcard_pattern`:
```mermaid
flowchart TD
A["1. Request arrives for claude-sonnet-4-6"] --> B["2. Auth check: can this key call this model?
proxy/auth/auth_checks.py"]
B --> C["3. Key has models=['anthropic/*']
→ wildcard match attempted"]
C --> D["4. get_llm_provider('claude-sonnet-4-6')
checks litellm.anthropic_models set"]
D -->|"model IN set"| E["5a. ✅ Provider = 'anthropic'
→ 'anthropic/claude-sonnet-4-6' matches 'anthropic/*'"]
D -->|"model NOT IN set"| F["5b. ❌ Provider unknown
→ exception raised → wildcard returns False"]
E --> G["6. Request allowed"]
F --> H["6. 401: key not allowed to access model"]
style E fill:#d4edda,stroke:#28a745
style F fill:#f8d7da,stroke:#dc3545
style H fill:#f8d7da,stroke:#dc3545
style D fill:#fff3cd,stroke:#ffc107
```
`litellm.anthropic_models` is a Python `set` populated at import time by `add_known_models()`. It is the source `get_llm_provider()` consults to map a bare model name like `claude-sonnet-4-6` to the provider string `"anthropic"`.
---
## Root Cause
`add_known_models()` is called **once** at module import time. Both reload paths in `proxy_server.py` updated `litellm.model_cost` with the fresh map but never called `add_known_models()` again:
```python
# Before the fix — both reload paths looked like this:
new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
litellm.model_cost = new_model_cost_map # ✅ cost map updated
_invalidate_model_cost_lowercase_map() # ✅ cache cleared
# ❌ add_known_models() never called
# → litellm.anthropic_models still has the old set
# → new model not in the set
# → get_llm_provider() raises for the new model
# → wildcard match returns False
# → 401 for every request to the new model
```
The gap existed in two places:
1. `_check_and_reload_model_cost_map` — the periodic automatic reload (every 10 s)
2. The `/reload/model_cost_map` admin endpoint — the manual reload
**Timeline:**
1. New model (`claude-sonnet-4-6`) added to `model_prices_and_context_window.json`
2. Admin triggers cost map reload via UI → `litellm.model_cost` updated
3. Users with `anthropic/*` wildcard keys attempt requests to `claude-sonnet-4-6`
4. `get_llm_provider('claude-sonnet-4-6')` raises → wildcard returns False → 401
5. Admin reloads cost map again — same result (root cause not addressed)
6. ~3 hours of investigation → root cause identified → fix deployed
---
## The Fix
After each reload, `add_known_models()` is called with the freshly fetched map passed explicitly. Passing the map directly (rather than relying on the module-level reference) removes any ambiguity about which dict is iterated:
```python
# After the fix — both reload paths now do:
new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
litellm.model_cost = new_model_cost_map
_invalidate_model_cost_lowercase_map()
litellm.add_known_models(model_cost_map=new_model_cost_map) # ✅ sets repopulated
```
`add_known_models()` was also updated to accept an optional explicit map so callers cannot accidentally iterate a stale module-level reference:
```python
# Before
def add_known_models():
for key, value in model_cost.items(): # reads module global — ambiguous after reload
...
# After
def add_known_models(model_cost_map: Optional[Dict] = None):
_map = model_cost_map if model_cost_map is not None else model_cost
for key, value in _map.items(): # always iterates the map you just fetched
...
```
After the fix, the provider sets (`anthropic_models`, `open_ai_chat_completion_models`, etc.) are always consistent with `litellm.model_cost` immediately after every reload. New models become accessible via wildcard rules without any proxy restart.
---
## Remediation
| # | Action | Status | Code |
|---|---|---|---|
| 1 | Call `add_known_models(model_cost_map=...)` in the periodic reload path | ✅ Done | [`proxy_server.py#L4393`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L4393) |
| 2 | Call `add_known_models(model_cost_map=...)` in the `/reload/model_cost_map` endpoint | ✅ Done | [`proxy_server.py#L11904`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L11904) |
| 3 | Update `add_known_models()` to accept an explicit map parameter | ✅ Done | [`__init__.py#L617`](https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py#L617) |
| 4 | Regression test: `add_known_models(model_cost_map=...)` populates provider sets | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
| 5 | Regression test: `anthropic/*` wildcard grants/denies access correctly after reload | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
---

175
docs/my-website/blog/gemini_3_1_flash_lite/index.md Normal file
View file

@ -0,0 +1,175 @@
---
slug: gemini_3_1_flash_lite_preview
title: "DAY 0 Support: Gemini 3.1 Flash Lite Preview on LiteLLM"
date: 2026-03-03T08:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Guide to using Gemini 3.1 Flash Lite Preview on LiteLLM Proxy and SDK with day 0 support."
tags: [gemini, day 0 support, llms, supernova]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Gemini 3.1 Flash Lite Preview Day 0 Support
LiteLLM now supports `gemini-3.1-flash-lite-preview` with full day 0 support!
:::note
If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
:::
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.80.8-stable.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==v1.80.8-stable.1
```
</TabItem>
</Tabs>
## What's New
Supports all four thinking levels:
- **MINIMAL**: Ultra-fast responses with minimal reasoning
- **LOW**: Simple instruction following
- **MEDIUM**: Balanced reasoning for complex tasks
- **HIGH**: Maximum reasoning depth (dynamic)
---
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
**Basic Usage**
```python
from litellm import completion
response = completion(
model="gemini/gemini-3.1-flash-lite-preview",
messages=[{"role": "user", "content": "Extract key entities from this text: ..."}],
)
print(response.choices[0].message.content)
```
**With Thinking Levels**
```python
from litellm import completion
# Use MEDIUM thinking for complex reasoning tasks
response = completion(
model="gemini/gemini-3.1-flash-lite-preview",
messages=[{"role": "user", "content": "Analyze this dataset and identify patterns"}],
reasoning_effort="medium", # low, medium , high
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gemini-3.1-flash-lite
litellm_params:
model: gemini/gemini-3.1-flash-lite-preview
api_key: os.environ/GEMINI_API_KEY
# Or use Vertex AI
- model_name: vertex-gemini-3.1-flash-lite
litellm_params:
model: vertex_ai/gemini-3.1-flash-lite-preview
vertex_project: your-project-id
vertex_location: us-central1
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
```
**3. Make requests**
```bash
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-d '{
"model": "gemini-3.1-flash-lite",
"messages": [{"role": "user", "content": "Extract structured data from this text"}],
"reasoning_effort": "low"
}'
```
</TabItem>
</Tabs>
---
## Supported Endpoints
LiteLLM provides **full end-to-end support** for Gemini 3.1 Flash Lite Preview on:
- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint
All endpoints support:
- Streaming and non-streaming responses
- Function calling with thought signatures
- Multi-turn conversations
- All Gemini 3-specific features (thinking levels, thought signatures)
- Full multimodal support (text, image, audio, video)
---
## `reasoning_effort` Mapping for Gemini 3.1
LiteLLM automatically maps OpenAI's `reasoning_effort` parameter to Gemini's `thinkingLevel`:
| reasoning_effort | thinking_level | Use Case |
|------------------|----------------|----------|
| `minimal` | `minimal` | Ultra-fast responses, simple queries |
| `low` | `low` | Basic instruction following |
| `medium` | `medium` | Balanced reasoning for moderate complexity |
| `high` | `high` | Maximum reasoning depth, complex problems |
| `disable` | `minimal` | Disable extended reasoning |
| `none` | `minimal` | No extended reasoning |

169
docs/my-website/blog/gemini_embedding_2_multimodal/index.md Normal file
View file

@ -0,0 +1,169 @@
---
slug: gemini_embedding_2_multimodal
title: "Gemini Embedding 2 Preview: Multimodal Embeddings on LiteLLM"
date: 2025-03-11T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
description: "Generate embeddings from text, images, audio, video, and PDFs with gemini-embedding-2-preview on LiteLLM via Gemini API and Vertex AI."
tags: [gemini, embeddings, multimodal, vertex ai]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Gemini Embedding 2 Preview: Multimodal Embeddings
LiteLLM now supports **multimodal embeddings** with `gemini-embedding-2-preview`—generating a single embedding from a mix of text, images, audio, video, and PDF content. Available via both the **Gemini API** (API key) and **Vertex AI** (GCP credentials).
## Supported Input Types
| Modality | Supported Formats |
|----------|-------------------|
| **Text** | Plain text |
| **Image** | PNG, JPEG |
| **Audio** | MP3, WAV |
| **Video** | MP4, MOV |
| **Documents** | PDF |
## Input Formats
LiteLLM accepts three input formats for multimodal content:
1. **Data URIs** – Base64-encoded inline: `data:image/png;base64,<encoded_data>`
2. **GCS URLs** – Cloud Storage paths (Vertex AI): `gs://bucket/path/to/file.png`
3. **Gemini File References** – Pre-uploaded files (Gemini API): `files/abc123`
## Quick Start
<Tabs>
<TabItem value="gemini" label="Gemini API">
```python
from litellm import embedding
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
# Text + Image (base64)
response = embedding(
model="gemini/gemini-embedding-2-preview",
input=[
"The food was delicious and the waiter...",
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
],
)
print(response)
```
</TabItem>
<TabItem value="vertex" label="Vertex AI">
```python
import litellm
from litellm import embedding
litellm.vertex_project = "your-project-id"
litellm.vertex_location = "us-central1"
# Text + Image (GCS URL)
response = embedding(
model="vertex_ai/gemini-embedding-2-preview",
input=[
"Describe this image",
"gs://my-bucket/images/photo.png"
],
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Config (config.yaml)**
```yaml
model_list:
- model_name: gemini-embedding-2-preview
litellm_params:
model: gemini/gemini-embedding-2-preview
api_key: os.environ/GEMINI_API_KEY
- model_name: vertex-gemini-embedding-2-preview
litellm_params:
model: vertex_ai/gemini-embedding-2-preview
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: os.environ/VERTEXAI_LOCATION
general_settings:
master_key: sk-1234
```
**2. Start proxy**
```bash
litellm --config config.yaml
```
**3. Call embeddings**
```bash
curl -X POST http://localhost:4000/embeddings \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gemini-embedding-2-preview",
"input": [
"The food was delicious and the waiter...",
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
]
}'
```
</TabItem>
</Tabs>
## Input Format Examples
| Format | Example | Provider |
|--------|---------|----------|
| **Data URI** | `data:image/png;base64,...` | Gemini, Vertex AI |
| **GCS URL** | `gs://bucket/path/image.png` | Vertex AI |
| **File reference** | `files/abc123` | Gemini API only |
### Supported MIME Types for Data URIs
- **Images:** `image/png`, `image/jpeg`
- **Audio:** `audio/mpeg`, `audio/wav`
- **Video:** `video/mp4`, `video/quicktime`
- **Documents:** `application/pdf`
### GCS URL MIME Inference
For Vertex AI, MIME types are inferred from file extensions:
- `.png` → `image/png`
- `.jpg` / `.jpeg` → `image/jpeg`
- `.mp3` → `audio/mpeg`
- `.wav` → `audio/wav`
- `.mp4` → `video/mp4`
- `.mov` → `video/quicktime`
- `.pdf` → `application/pdf`
## Optional Parameters
| Parameter | Description | Maps to |
|-----------|-------------|---------|
| `dimensions` | Output embedding size | `outputDimensionality` |
```python
response = embedding(
model="gemini/gemini-embedding-2-preview",
input=["text to embed"],
dimensions=768, # Optional: control output vector size
)
```

145
docs/my-website/blog/gpt_5_3_codex/index.md Normal file
View file

@ -0,0 +1,145 @@
---
slug: gpt_5_3_codex
title: "Day 0 Support: GPT-5.3-Codex"
date: 2026-02-24T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API."
tags: [openai, gpt-5.3-codex, codex, day 0 support]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items.
## Why `phase` matters for GPT-5.3-Codex
`phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses.
Reference: [Phase parameter docs](https://developers.openai.com/api/reference/overview)
Supported values:
- `null`
- `"commentary"`
- `"final_answer"`
Important:
- Persist assistant output items with `phase` exactly as returned.
- Send those assistant items back on the next turn.
- Do **not** add `phase` to user messages.
## Docker Image
```bash
docker pull ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3
```
## Usage
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gpt-5.3-codex
litellm_params:
model: openai/gpt-5.3-codex
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e ANTHROPIC_API_KEY=$OPENAI_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3 \
--config /app/config.yaml
```
**3. Test it**
```bash
curl -X POST "http://0.0.0.0:4000/v1/responses" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gpt-5.3-codex",
"input": "Write a Python script that checks if a number is prime."
}'
```
</TabItem>
</Tabs>
## Python Example: Persist `phase` with OpenAI Client + LiteLLM Base URL
```python
from openai import OpenAI
client = OpenAI(
base_url="http://0.0.0.0:4000/v1", # LiteLLM Proxy
api_key="your-litellm-api-key",
)
items = [] # Persist this per conversation/thread
def _item_get(item, key, default=None):
if isinstance(item, dict):
return item.get(key, default)
return getattr(item, key, default)
def run_turn(user_text: str):
global items
# User message: no phase field
items.append(
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": user_text}],
}
)
resp = client.responses.create(
model="gpt-5.3-codex",
input=items,
)
# Persist assistant output items verbatim, including phase
for out_item in (resp.output or []):
items.append(out_item)
# Optional: inspect latest phase for UI/telemetry routing
latest_phase = None
for out_item in reversed(resp.output or []):
if _item_get(out_item, "type") == "output_item.done" and _item_get(out_item, "phase") is not None:
latest_phase = _item_get(out_item, "phase")
break
return resp, latest_phase
```
## Notes
- Use `/v1/responses` for GPT Codex models.
- Preserve full assistant output history for best multi-turn behavior.
- If `phase` metadata is dropped during history reconstruction, output quality can degrade on long-running tasks.

97
docs/my-website/blog/gpt_5_4/index.md Normal file
View file

@ -0,0 +1,97 @@
---
slug: gpt_5_4
title: "Day 0 Support: GPT-5.4"
date: 2026-03-05T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "GPT-5.4 model support in LiteLLM"
tags: [openai, gpt-5.4, completion]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM now supports fully GPT-5.4!
## Docker Image
```bash
docker pull ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch
```
## Usage
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gpt-5.4
litellm_params:
model: openai/gpt-5.4
api_key: os.environ/OPENAI_API_KEY
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch \
--config /app/config.yaml
```
**3. Test it**
```bash
curl -X POST "http://0.0.0.0:4000/chat/completions" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gpt-5.4",
"messages": [
{"role": "user", "content": "Write a Python function to check if a number is prime."}
]
}'
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
from litellm import completion
response = completion(
model="openai/gpt-5.4",
messages=[
{"role": "user", "content": "Write a Python function to check if a number is prime."}
],
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
## Notes
- Restart your container to get the cost tracking for this model.
- Use `/responses` for better model performance.
- GPT-5.4 supports reasoning, function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.

132
docs/my-website/blog/httpx_cache_eviction_incident/index.md Normal file
View file

@ -0,0 +1,132 @@
---
slug: httpx-cache-eviction-incident
title: "Incident Report: Cache Eviction Closes In-Use httpx Clients"
date: 2026-02-27T10:00:00
authors:
- name: Ryan Crabbe
title: Performance Engineer, LiteLLM
url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
tags: [incident-report, caching, stability]
hide_table_of_contents: false
---
**Date:** February 27, 2026
**Duration:** ~6 days (Feb 21 merge -> Feb 27 fix)
**Severity:** High
**Status:** Resolved
> **Note:** This fix is available starting from LiteLLM `v1.81.14.rc.2` or higher.
## Summary
A change to improve Redis connection pool cleanup introduced a regression that closed **httpx clients** that were still actively being used by the proxy. The `LLMClientCache` (an in-memory TTL cache) stores both Redis clients *and* httpx clients under the same eviction policy. When a cache entry expired or was evicted, the new cleanup code called `aclose()`/`close()` on the evicted value which worked correctly for Redis clients, but destroyed httpx clients that other parts of the system still held references to and were actively using for LLM API calls.
**Impact:** Any proxy instance that hit the cache TTL (default 10 minutes) or capacity limit (200 entries) would have its httpx clients closed out from under it, causing requests to LLM providers to fail with connection errors.
---
## Background
`LLMClientCache` extends `InMemoryCache` and is used to cache SDK clients (OpenAI, Anthropic, etc.) to avoid re-creating them on every request. These clients are keyed by configuration + event loop ID. The cache has:
- **Max size:** 200 entries
- **Default TTL:** 10 minutes
When the cache is full or entries expire, `InMemoryCache.evict_cache()` calls `_remove_key()` to drop entries.
The cached values are a mix of:
- **Redis/async Redis clients** — owned exclusively by the cache, safe to close on eviction
- **httpx-backed SDK clients** (OpenAI, Anthropic, etc.) — shared references, still in use by router/model instances
---
## Root Cause
[PR #21717](https://github.com/BerriAI/litellm/pull/21717) overrode `_remove_key()` in `LLMClientCache` to close async clients on eviction:
<details>
<summary>Problematic code added in PR #21717</summary>
```python
class LLMClientCache(InMemoryCache):
def _remove_key(self, key: str) -> None:
value = self.cache_dict.get(key)
super()._remove_key(key)
if value is not None:
close_fn = getattr(value, "aclose", None) or getattr(value, "close", None)
if close_fn and asyncio.iscoroutinefunction(close_fn):
try:
asyncio.get_running_loop().create_task(close_fn())
except RuntimeError:
pass
elif close_fn and callable(close_fn):
try:
close_fn()
except Exception:
pass
```
</details>
The intent was correct for Redis clients — prevent connection pool leaks when cached Redis clients expire. But `LLMClientCache` also stores httpx-backed SDK clients (e.g., `AsyncOpenAI`, `AsyncAnthropic`). These clients:
1. Have an `aclose()` method (inherited from httpx)
2. Are still held by references elsewhere in the codebase (router, model instances)
3. Were being closed without any check on whether they were still in use
So when the cache evicted an entry, it would call `aclose()` on an httpx client that was still being used for active LLM requests → closed transport → connection errors.
---
## The Fix
[PR #22247](https://github.com/BerriAI/litellm/pull/22247) removed the `_remove_key` override entirely:
<details>
<summary>The fix (PR #22247)</summary>
```diff
class LLMClientCache(InMemoryCache):
- def _remove_key(self, key: str) -> None:
- """Close async clients before evicting them to prevent connection pool leaks."""
- value = self.cache_dict.get(key)
- super()._remove_key(key)
- if value is not None:
- close_fn = getattr(value, "aclose", None) or getattr(
- value, "close", None
- )
- ...
-
def update_cache_key_with_event_loop(self, key):
```
</details>
The eviction now simply drops the reference and lets Python's GC handle cleanup, which is safe because:
- httpx clients that are still referenced elsewhere stay alive
- Unreferenced clients get cleaned up by GC naturally
The other improvements from PR #21717 were kept:
- **`max_connections` respected for URL-based Redis configs**, previously silently dropped
- **`disconnect()` now closes both sync and async Redis clients**, sync client was previously leaked
- **Connection pool passthrough**, when a pool is provided with a URL config, it's used directly instead of creating a duplicate
---
## Remediation
| Action | Status | Code |
|--------|--------|------|
| Remove `_remove_key` override that closes shared clients on eviction | ✅ Done | [PR #22247](https://github.com/BerriAI/litellm/pull/22247) |
| Add e2e test: evicted client still usable (capacity) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
| Add e2e test: expired client still usable (TTL) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
The e2e tests go through `get_async_httpx_client()` the same code path the proxy uses in production and assert the client is still functional after eviction. These run in CI on every PR against `main`. If anyone modifies `LLMClientCache` eviction behavior, overrides `_remove_key`, or adds any form of client cleanup on eviction, these tests will fail regardless of the implementation approach.

321
docs/my-website/blog/responses_api_encrypted_content_incident/index.md Normal file
View file

@ -0,0 +1,321 @@
---
slug: responses-api-encrypted-content-incident
title: "Incident Report: Encrypted Content Failures in Multi-Region Responses API Load Balancing"
date: 2026-02-24T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
tags: [incident-report, proxy, responses-api, load-balancing]
hide_table_of_contents: false
---
**Date:** Feb 24, 2026
**Duration:** Ongoing (until fix deployed)
**Severity:** High (for users load balancing Responses API across different API keys)
**Status:** Resolved
## Summary
When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or OpenAI organizations), follow-up requests containing encrypted content items (like `rs_...` reasoning items) would fail with:
```json
{
"error": {
"message": "The encrypted content for item rs_0d09d6e56879e76500699d6feee41c8197bd268aae76141f87 could not be verified. Reason: Encrypted content organization_id did not match the target organization.",
"type": "invalid_request_error",
"code": "invalid_encrypted_content"
}
}
```
Encrypted content items are cryptographically tied to the API key's organization that created them. When the router load balanced a follow-up request to a deployment with a different API key, decryption failed.
- **Responses API calls with encrypted content:** Complete failure when routed to wrong deployment
- **Initial requests:** Unaffected — only follow-up requests containing encrypted items failed
- **Other API endpoints:** No impact — chat completions, embeddings, etc. functioned normally
{/* truncate */}
---
## Background
OpenAI's Responses API can return encrypted "reasoning items" (with IDs like `rs_...`) that contain intermediate reasoning steps. These items are encrypted with the organization's key and can only be decrypted by the same organization's API key.
When load balancing across deployments with different API keys, the existing affinity mechanisms were insufficient:
- **`responses_api_deployment_check`**: Requires `previous_response_id` which some clients (like Codex) don't provide
- **`deployment_affinity`**: Too broad — pins *all* requests from a user to one deployment, reducing effective quota by the number of users
- **`session_affinity`**: Requires explicit session IDs and still reduces quota
```mermaid
flowchart TD
A["1. Initial request to Responses API
router.aresponses()"] --> B["2. Router load balances to Deployment A
(API Key 1, Azure East US)"]
B --> C["3. Response contains encrypted item
rs_abc123 (encrypted with Org 1 key)"]
C --> D["4. Follow-up request includes rs_abc123 in input"]
D --> E["5. Router load balances to Deployment B
(API Key 2, Azure West Europe)"]
E -->|"Different API key"| F["6. ❌ Deployment B cannot decrypt rs_abc123
Error: invalid_encrypted_content"]
D -.->|"With encrypted_content_affinity"| G["5b. Router detects rs_abc123 was created by Deployment A"]
G --> H["6b. ✅ Routes to Deployment A (bypasses rate limits)
Request succeeds"]
style F fill:#f8d7da,stroke:#dc3545
style H fill:#d4edda,stroke:#28a745
style E fill:#fff3cd,stroke:#ffc107
style G fill:#d4edda,stroke:#28a745
```
---
## Root Cause
LiteLLM's router had no mechanism to track which deployment created specific encrypted content items and route follow-up requests accordingly. The router treated all deployments as interchangeable, leading to decryption failures when encrypted content crossed organizational boundaries.
**The Problem Flow:**
1. User calls `router.aresponses()` with model `gpt-5.1-codex`
2. Router load balances to Deployment A (Azure East US, API Key 1)
3. Response contains encrypted reasoning item `rs_abc123` (encrypted with Org 1's key)
4. User makes follow-up request with `rs_abc123` in the input
5. Router load balances to Deployment B (Azure West Europe, API Key 2)
6. Deployment B tries to decrypt `rs_abc123` with Org 2's key → **fails**
**Why Existing Solutions Didn't Work:**
- **`previous_response_id`**: Not provided by all clients (e.g., Codex)
- **`deployment_affinity`**: Pins *all* user requests to one deployment → reduces quota to 1/N where N = number of deployments
- **`session_affinity`**: Requires explicit session management and still reduces quota
**Timeline:**
1. Users configured multi-region Responses API load balancing with different API keys
2. Initial requests succeeded, but follow-up requests with encrypted content failed intermittently
3. Error rate correlated with number of deployments (more deployments = higher chance of routing to wrong one)
4. Investigation revealed encrypted content was organization-bound
5. Existing affinity mechanisms deemed unsuitable (quota reduction, missing `previous_response_id`)
6. New solution designed and implemented: `encrypted_content_affinity`
---
## The Fix
Implemented a new `encrypted_content_affinity` pre-call check that intelligently tracks encrypted content and routes follow-up requests **only when necessary**.
### Implementation
**1. Encoding `model_id` into output items** ([`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py))
The same approach used for `previous_response_id` affinity — no cache needed. When a response contains output items with `encrypted_content`, LiteLLM encodes the originating deployment's `model_id` in **two places** for redundancy:
1. **Into the item ID** (if present): `rs_abc123` → `encitem_{base64("litellm:model_id:{model_id};item_id:rs_abc123")}`
2. **Into the encrypted_content itself**: Wraps the content with `litellm_enc:{base64("model_id:{model_id}")};{original_encrypted_content}`
```python
# Encoding item IDs (when present)
def _build_encrypted_item_id(model_id: str, item_id: str) -> str:
assembled = f"litellm:model_id:{model_id};item_id:{item_id}"
encoded = base64.b64encode(assembled.encode("utf-8")).decode("utf-8")
return f"encitem_{encoded}"
# Wrapping encrypted_content (always, for redundancy)
def _wrap_encrypted_content_with_model_id(encrypted_content: str, model_id: str) -> str:
metadata = f"model_id:{model_id}"
encoded_metadata = base64.b64encode(metadata.encode("utf-8")).decode("utf-8")
return f"litellm_enc:{encoded_metadata};{encrypted_content}"
```
**Why wrap encrypted_content directly?** Some clients (like Codex) don't consistently send item IDs in follow-up requests, but they always send the `encrypted_content` itself. By embedding `model_id` into the content, affinity works even when IDs are missing.
**Streaming responses:** The wrapping logic is applied to both:
- Final response objects (non-streaming)
- Individual streaming events (`response.output_item.added`, `response.output_item.done`)
This ensures clients receiving streaming responses get wrapped content they can send back.
Before forwarding to the upstream provider, LiteLLM restores the original item IDs and unwraps encrypted_content so the provider never sees the encoded form:
```python
# In responses/main.py — before calling the handler
input = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(input)
```
**2. `EncryptedContentAffinityCheck` — routing only** ([`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py))
No `async_log_success_event` or cache lookups — the `model_id` is decoded directly from the item ID or encrypted_content:
```python
class EncryptedContentAffinityCheck(CustomLogger):
async def async_filter_deployments(self, model, healthy_deployments, ...):
"""Extract model_id from input items (ID or encrypted_content) and pin to that deployment."""
for item in request_kwargs.get("input", []):
# Try to extract model_id from two sources:
model_id = self._extract_model_id_from_input(item)
if model_id:
deployment = self._find_deployment_by_model_id(
healthy_deployments, model_id
)
if deployment:
request_kwargs["_encrypted_content_affinity_pinned"] = True
return [deployment]
return healthy_deployments
def _extract_model_id_from_input(self, item: dict) -> Optional[str]:
"""Extract model_id from either encoded ID or wrapped encrypted_content."""
# 1. Try decoding from item ID (if present)
item_id = item.get("id", "")
if item_id:
decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id)
if decoded:
return decoded["model_id"]
# 2. Try unwrapping from encrypted_content (fallback for clients that omit IDs)
encrypted_content = item.get("encrypted_content", "")
if encrypted_content and encrypted_content.startswith("litellm_enc:"):
model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
encrypted_content
)
return model_id
return None
```
**3. Rate Limit Bypass** ([`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py))
When encrypted content requires a specific deployment, RPM/TPM limits are bypassed (the request would fail on any other deployment anyway):
```python
# In async_get_available_deployment, after filtering healthy deployments:
if (
request_kwargs.get("_encrypted_content_affinity_pinned")
and len(healthy_deployments) == 1
):
return healthy_deployments[0] # Bypass routing strategy (RPM/TPM checks)
```
**3. Configuration**
```yaml
router_settings:
routing_strategy: usage-based-routing-v2
enable_pre_call_checks: true
optional_pre_call_checks:
- encrypted_content_affinity
deployment_affinity_ttl_seconds: 86400 # 24 hours
```
### Key Benefits
✅ **No quota reduction**: Only pins requests containing encrypted items
✅ **Bypasses rate limits**: When encrypted content requires a specific deployment, RPM/TPM limits don't block it
✅ **No `previous_response_id` required**: Works by encoding `model_id` directly into the item ID
✅ **No cache required**: `model_id` is decoded on-the-fly from the item ID — no Redis, no TTL
✅ **Globally safe**: Can be enabled for all models; non-Responses-API calls are unaffected
✅ **Surgical precision**: Normal requests continue to load balance freely
---
## Remediation
| # | Action | Status | Code |
|---|---|---|---|
| 1 | Encode `model_id` into encrypted-content item IDs on response | ✅ Done | [`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py) |
| 2 | Restore original item IDs before forwarding to upstream provider | ✅ Done | [`responses/main.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/main.py) |
| 3 | `EncryptedContentAffinityCheck`: decode item IDs to route (no cache) | ✅ Done | [`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py) |
| 4 | Add `encrypted_content_affinity` to `OptionalPreCallChecks` type | ✅ Done | [`types/router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/types/router.py) |
| 5 | Implement rate limit bypass for affinity-pinned requests | ✅ Done | [`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py) |
| 6 | Unit tests: encoding/decoding utilities, routing, RPM bypass | ✅ Done | [`test_encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py) |
| 7 | Documentation: Responses API guide, load balancing guide, config reference | ✅ Done | [Docs](https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing) |
| 8 | **[Mar 3]** Fix streaming events to wrap encrypted_content | ✅ Done | [`responses/streaming_iterator.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/streaming_iterator.py) |
---
## Follow-up Fix: Streaming Responses (Mar 3, 2026)
### The Issue
After the initial fix was deployed, users reported that the `invalid_encrypted_content` error **still occurred** when using streaming responses with clients like Codex. Investigation revealed:
- ✅ Non-streaming responses: `encrypted_content` was correctly wrapped with `litellm_enc:` prefix
- ❌ Streaming responses: Individual `response.output_item.added` and `response.output_item.done` events contained **raw, unwrapped** `encrypted_content`
Since Codex and other clients consume responses as streams, they received unwrapped content in these events and sent it back in follow-up requests, causing the affinity check to fail.
### The Root Cause
The `_update_encrypted_content_item_ids_in_response` function only modified the **final** response object, which is used for non-streaming responses. For streaming responses, individual chunks are processed by `ResponsesAPIStreamingIterator._process_chunk`, which was **not** applying the wrapping logic to streaming events.
### The Fix
Modified `litellm/litellm/responses/streaming_iterator.py` to wrap `encrypted_content` in streaming events:
```python
# In ResponsesAPIStreamingIterator._process_chunk
if (
self.litellm_metadata
and self.litellm_metadata.get("encrypted_content_affinity_enabled")
):
event_type = getattr(openai_responses_api_chunk, "type", None)
if event_type in (
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
):
item = getattr(openai_responses_api_chunk, "item", None)
if item:
encrypted_content = getattr(item, "encrypted_content", None)
if encrypted_content and isinstance(encrypted_content, str):
model_id = (
self.litellm_metadata.get("model_info", {}).get("id")
if self.litellm_metadata
else None
)
if model_id:
wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
encrypted_content, model_id
)
setattr(item, "encrypted_content", wrapped_content)
```
This ensures that **all** `encrypted_content` sent to clients (streaming or non-streaming) is wrapped with `model_id` metadata, enabling consistent affinity routing.
---
## Migration Guide
### Before (Using `deployment_affinity`)
```yaml
router_settings:
optional_pre_call_checks:
- deployment_affinity # ❌ Reduces quota by number of users
```
**Problem:** All requests from a user pin to one deployment, reducing effective quota to 1/N.
### After (Using `encrypted_content_affinity`)
```yaml
router_settings:
optional_pre_call_checks:
- encrypted_content_affinity # ✅ Only pins requests with encrypted content
```
**Benefit:** Normal requests load balance freely, only encrypted content requests pin when necessary.
---

1
docs/my-website/docs/a2a.md
View file

@ -20,6 +20,7 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque
| Logging | ✅ |
| Load Balancing | ✅ |
| Streaming | ✅ |
| [Iteration Budgets](a2a_iteration_budgets) | ✅ |
:::tip

252
docs/my-website/docs/a2a_agent_headers.md Normal file
View file

@ -0,0 +1,252 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# A2A Agent Authentication Headers
Forward authentication credentials (Bearer tokens, API keys, etc.) from clients to backend A2A agents.
## Overview
When LiteLLM proxies a request to a backend A2A agent, the agent may require its own authentication headers. There are three ways to supply them:
| Method | Who configures | How it works |
|---|---|---|
| **Static headers** | Admin (UI / API) | Always sent, regardless of client request |
| **Forward client headers** | Admin (UI / API) | Header names to extract from client request and forward |
| **Convention-based** | Client (no admin config) | Client sends `x-a2a-{agent_name}-{header}` — automatically routed |
All three methods can be combined. **Static headers always win** on key conflicts.
---
## Method 1 — Static Headers
Admin-configured headers that are always sent to the backend agent. Use this for server-to-server tokens or internal credentials that clients should never see or override.
<Tabs>
<TabItem value="ui" label="UI">
1. Go to **Agents** in the LiteLLM dashboard.
2. Create or edit an agent.
3. Open the **Authentication Headers** panel.
4. Under **Static Headers**, click **Add Static Header** and fill in the header name and value.
</TabItem>
<TabItem value="api" label="REST API">
```bash
curl -X POST http://localhost:4000/v1/agents \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"agent_name": "my-agent",
"agent_card_params": { ... },
"static_headers": {
"Authorization": "Bearer internal-server-token",
"X-Internal-Service": "litellm-proxy"
}
}'
```
To update an existing agent:
```bash
curl -X PATCH http://localhost:4000/v1/agents/{agent_id} \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"static_headers": {
"Authorization": "Bearer new-token"
}
}'
```
</TabItem>
</Tabs>
**Client call — no special headers needed:**
```bash
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "Content-Type: application/json" \
-d '{
"jsonrpc": "2.0", "id": "1", "method": "message/send",
"params": { "message": { "role": "user", "parts": [{"kind": "text", "text": "Hello"}], "messageId": "msg-1" } }
}'
```
The backend agent receives `Authorization: Bearer internal-server-token` without the client ever knowing the value.
---
## Method 2 — Forward Client Headers
Admin specifies a list of header **names**. When the client sends a request that includes those headers, LiteLLM extracts their values and forwards them to the backend agent. The client controls the values; the admin controls which headers are eligible to be forwarded.
<Tabs>
<TabItem value="ui" label="UI">
1. Go to **Agents** in the LiteLLM dashboard.
2. Create or edit an agent.
3. Open the **Authentication Headers** panel.
4. Under **Forward Client Headers**, type header names and press **Enter** (e.g. `x-api-key`, `Authorization`).
</TabItem>
<TabItem value="api" label="REST API">
```bash
curl -X POST http://localhost:4000/v1/agents \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"agent_name": "my-agent",
"agent_card_params": { ... },
"extra_headers": ["x-api-key", "x-user-token"]
}'
```
</TabItem>
</Tabs>
**Client call — include the forwarded headers:**
```bash
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "x-api-key: user-secret-value" \
-H "Content-Type: application/json" \
-d '{ ... }'
```
The backend agent receives `x-api-key: user-secret-value`.
:::note
Header name matching is **case-insensitive**. If the client sends `X-API-Key` and `extra_headers` lists `x-api-key`, they match.
:::
---
## Method 3 — Convention-Based Forwarding
Clients can forward headers to a specific agent without any admin pre-configuration by using the naming convention:
```
x-a2a-{agent_name_or_id}-{header_name}: value
```
LiteLLM parses these headers automatically and routes them to the matching agent only.
**Examples:**
| Client header sent | Agent name/ID | Forwarded as |
|---|---|---|
| `x-a2a-my-agent-authorization: Bearer tok` | `my-agent` | `authorization: Bearer tok` |
| `x-a2a-my-agent-x-api-key: secret` | `my-agent` | `x-api-key: secret` |
| `x-a2a-abc123-authorization: Bearer tok` | agent ID `abc123` | `authorization: Bearer tok` |
```bash
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "x-a2a-my-agent-authorization: Bearer agent-specific-token" \
-H "Content-Type: application/json" \
-d '{ ... }'
```
The `x-a2a-other-agent-authorization` header sent in the same request is **not** forwarded to `my-agent` — it is silently ignored.
:::tip Matches both agent name and agent ID
Both the human-readable name (e.g. `my-agent`) and the UUID (e.g. `abc123-...`) are valid. Use whichever is convenient for the client.
:::
---
## Merge Precedence
When multiple methods supply the same header name, **static headers win**:
```
dynamic (forwarded/convention) → merged ← static (overlays, wins)
```
Example:
| Source | `Authorization` value |
|---|---|
| Client sends (via `extra_headers` or convention) | `Bearer client-token` |
| Admin-configured `static_headers` | `Bearer server-token` |
| **What the backend agent receives** | **`Bearer server-token`** |
This ensures admin-controlled credentials cannot be overridden by client requests.
---
## Combining All Three Methods
```bash
# Register agent with static + forwarded headers
curl -X POST http://localhost:4000/v1/agents \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"agent_name": "my-agent",
"agent_card_params": { ... },
"static_headers": {
"X-Internal-Token": "secret123"
},
"extra_headers": ["x-user-id"]
}'
# Client call using all three mechanisms
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "x-user-id: user-42" \
-H "x-a2a-my-agent-x-request-id: req-abc" \
-H "Content-Type: application/json" \
-d '{ ... }'
```
The backend agent receives:
```
X-Internal-Token: secret123 ← static header (always)
x-user-id: user-42 ← forwarded (in extra_headers)
x-request-id: req-abc ← convention-based (x-a2a-my-agent-*)
X-LiteLLM-Trace-Id: <uuid> ← LiteLLM internal
X-LiteLLM-Agent-Id: <agent-id> ← LiteLLM internal
```
---
## Header Isolation
Each agent invocation uses an isolated HTTP connection. Headers configured for agent A are **never** sent to agent B, even if both agents are running and receiving requests simultaneously.
---
## API Reference
### `POST /v1/agents` / `PATCH /v1/agents/{agent_id}`
| Field | Type | Description |
|---|---|---|
| `static_headers` | `object` | `{"Header-Name": "value"}` — always forwarded |
| `extra_headers` | `string[]` | Header names to extract from client request and forward |
### Agent Response
Both fields are returned in `GET /v1/agents` and `GET /v1/agents/{agent_id}`:
```json
{
"agent_id": "...",
"agent_name": "my-agent",
"static_headers": { "X-Internal-Token": "secret123" },
"extra_headers": ["x-user-id"],
...
}
```
:::caution
`static_headers` values are stored in the database and returned by the API. Treat them as you would any credential — do not store sensitive long-lived tokens here if your API is publicly accessible. Consider using short-lived tokens or environment-injected secrets instead.
:::

188
docs/my-website/docs/a2a_iteration_budgets.md Normal file
View file

@ -0,0 +1,188 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Agent Iteration Budgets
Control runaway costs from agentic loops with per-session iteration and budget caps.
## Overview
When agents run agentic loops, they can make unbounded LLM calls, causing unexpected costs. LiteLLM provides two controls:
| Control | Description |
|---------|-------------|
| **Max Iterations** | Hard cap on the number of LLM calls per session |
| **Max Budget Per Session** | Dollar cap per session (identified by `x-litellm-trace-id`) |
Both controls require a `session_id` (sent via `x-litellm-trace-id` header or `metadata.session_id`) to track calls within a session.
## Trace-ID Enforcement
LiteLLM supports two independent trace-id flags, configured in `litellm_params` on the agent:
| Flag | Description |
|------|-------------|
| `require_trace_id_on_calls_to_agent` | Requires callers invoking this agent to include `x-litellm-trace-id`. Use when the agent should only be called as a sub-agent with a trace context. Returns **400** if missing. |
| `require_trace_id_on_calls_by_agent` | Requires all LLM/MCP calls made **by** this agent (via its virtual key) to include `x-litellm-trace-id`. This is what enables `max_iterations` and `max_budget_per_session` tracking. Returns **400** if missing. |
## Configuring via UI
When creating an agent in the LiteLLM Admin UI:
1. Navigate to the **Agents** tab and click **Add Agent**
2. In the **Agent Settings** step, expand the **Tracing** section
3. Toggle **Require x-litellm-trace-id on calls BY this agent** to enable session tracking
4. Set **Max Iterations** to cap the number of LLM calls per session
5. Set **Max Budget Per Session ($)** to cap spend per session
The trace-id flags are stored on the agent's `litellm_params`. Budget controls (`max_iterations`, `max_budget_per_session`) are stored in the virtual key's metadata.
## Configuring via API
Set trace-id enforcement on the agent itself:
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent with budget controls",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"litellm_params": {
"require_trace_id_on_calls_to_agent": true,
"require_trace_id_on_calls_by_agent": true
}
}'
```
Budget controls are set on the agent's `litellm_params` (not on individual keys), so they apply across all keys for the agent:
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent with budget controls",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"litellm_params": {
"require_trace_id_on_calls_by_agent": true,
"max_iterations": 25,
"max_budget_per_session": 5.00
}
}'
```
## How It Works
### Session Tracking
Callers identify their session by including a `session_id` in one of these ways:
- **Header**: `x-litellm-trace-id: my-session-123`
- **Metadata**: `{"metadata": {"session_id": "my-session-123"}}`
### Max Iterations
When `max_iterations` is set in agent `litellm_params`:
- Each LLM call for a session increments a counter
- When the counter exceeds `max_iterations`, the request receives a **429 Too Many Requests**
- Counters expire after 1 hour by default (configurable via `LITELLM_MAX_ITERATIONS_TTL` env var)
### Max Budget Per Session
When `max_budget_per_session` is set in agent `litellm_params`:
- After each successful LLM call, the response cost is accumulated for the session
- Before each call, the accumulated spend is checked against the budget
- When spend exceeds the budget, the request receives a **429 Too Many Requests**
- Session spend counters expire after 1 hour by default (configurable via `LITELLM_MAX_BUDGET_PER_SESSION_TTL` env var)
## Example
Create an agent with max 25 iterations and a $5 budget cap:
<Tabs>
<TabItem value="ui" label="Via UI">
1. Go to **Agents** → **Add Agent**
2. Configure your agent (name, model, etc.)
3. In **Agent Settings**, expand the **Tracing** section
4. Toggle on **Require x-litellm-trace-id on calls BY this agent**
5. Set **Max Iterations** to `25`
6. Set **Max Budget Per Session** to `5.00`
7. Proceed to create a new key for the agent
8. Click **Create Agent**
</TabItem>
<TabItem value="api" label="Via API">
```bash
# 1. Create the agent with trace-id enforcement
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent with budget controls",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"litellm_params": {
"require_trace_id_on_calls_by_agent": true
}
}'
# 2. Create a key for the agent
curl -X POST 'http://localhost:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_id": "<agent_id_from_step_1>",
"key_alias": "my-research-agent-key"
}'
```
</TabItem>
</Tabs>
### Making Calls with Session Tracking
```bash
curl -X POST 'http://localhost:4000/chat/completions' \
-H 'Authorization: Bearer sk-agent-key-xxx' \
-H 'x-litellm-trace-id: session-abc-123' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
After 25 calls or $5 spent within this session, subsequent requests will receive:
```json
{
"error": {
"message": "Session budget exceeded for session session-abc-123. Current spend: $5.0032, max_budget_per_session: $5.00.",
"type": "budget_exceeded",
"code": 429
}
}
```
## Environment Variables
| Variable | Default | Description |
|----------|---------|-------------|
| `LITELLM_MAX_ITERATIONS_TTL` | `3600` (1 hour) | TTL in seconds for session iteration counters |
| `LITELLM_MAX_BUDGET_PER_SESSION_TTL` | `3600` (1 hour) | TTL in seconds for session budget counters |

29
docs/my-website/docs/adding_provider/generic_guardrail_api.md
View file

@ -244,6 +244,35 @@ litellm_settings:
language: "en"
```
### Static and dynamic headers
You can send two kinds of headers to your guardrail endpoint:
- **Static headers** (`headers`): A key/value map sent with **every** request to your guardrail. Use this for fixed values (e.g. API keys, `X-Service-Name`). Configure in `litellm_params`:
```yaml
litellm_params:
guardrail: generic_guardrail_api
api_base: https://your-guardrail-api.com
headers:
X-Service-Name: "my-app"
X-API-Key: "secret"
```
- **Dynamic headers** (`extra_headers`): A list of **header names** that are forwarded from the **client request** to your guardrail. Only headers in this list (plus a small default allowlist such as `x-litellm-*`) have their values sent; others are sent as `[present]`. Use this to pass through client-provided headers (e.g. `x-request-id`, `x-correlation-id`). Configure in `litellm_params`:
```yaml
litellm_params:
guardrail: generic_guardrail_api
api_base: https://your-guardrail-api.com
extra_headers:
- x-request-id
- x-correlation-id
- x-custom-auth
```
This mirrors the [MCP static and extra headers](/docs/mcp#forwarding-custom-headers-to-mcp-servers) behavior.
### Example: Pillar Security
[Pillar Security](https://pillar.security) uses the Generic Guardrail API to provide comprehensive AI security scanning including prompt injection protection, PII/PCI detection, secret detection, and content moderation.

1
docs/my-website/docs/anthropic_count_tokens.md
View file

@ -138,6 +138,7 @@ The `/v1/messages/count_tokens` endpoint automatically routes to the appropriate
| Provider | Token Counting Method |
|----------|----------------------|
| Anthropic | [Anthropic Token Counting API](https://docs.anthropic.com/en/docs/build-with-claude/token-counting) |
| OpenAI | [OpenAI Responses API `/input_tokens`](https://platform.openai.com/docs/api-reference/responses/input-tokens) — see [Token Counting](./count_tokens.md) |
| Vertex AI (Claude) | Vertex AI Partner Models Token Counter |
| Bedrock (Claude) | AWS Bedrock CountTokens API |
| Gemini | Google AI Studio countTokens API |

120
docs/my-website/docs/anthropic_unified/messages_to_responses_mapping.md Normal file
View file

@ -0,0 +1,120 @@
# v1/messages → /responses Parameter Mapping
When you send a request to `/v1/messages` targeting an OpenAI or Azure model, LiteLLM internally routes it through the OpenAI Responses API. This page documents exactly how every parameter gets translated in both directions.
The transformation lives in `litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py`.
## Request: Anthropic → Responses API
### Top-level parameters
| Anthropic (`/v1/messages`) | Responses API | Notes |
|---|---|---|
| `model` | `model` | Passed through as-is |
| `messages` | `input` | Structurally transformed — see the messages section below |
| `system` (string) | `instructions` | Passed as a plain string |
| `system` (list of content blocks) | `instructions` | Text blocks are joined with `\n`; non-text blocks are ignored |
| `max_tokens` | `max_output_tokens` | Renamed |
| `temperature` | `temperature` | Passed through as-is |
| `top_p` | `top_p` | Passed through as-is |
| `tools` | `tools` | Format-translated — see the tools section below |
| `tool_choice` | `tool_choice` | Type-remapped — see the tool_choice section below |
| `thinking` | `reasoning` | Budget tokens mapped to effort level — see the thinking section below |
| `output_format` or `output_config.format` | `text` | Wrapped as `{"format": {"type": "json_schema", "name": "structured_output", "schema": ..., "strict": true}}` |
| `context_management` | `context_management` | Converted from Anthropic dict to OpenAI array format — see the context_management section below |
| `metadata.user_id` | `user` | Extracted from the metadata object and truncated to 64 characters |
| `stop_sequences` | ❌ Not mapped | Dropped silently |
| `top_k` | ❌ Not mapped | Dropped silently |
| `speed` | ❌ Not mapped | Only used to set Anthropic beta headers on the native path |
### How messages get converted
Each Anthropic message is expanded into one or more Responses API input items. The key difference is that `tool_result` and `tool_use` blocks become **top-level items** in the input array rather than being nested inside a message.
| Anthropic message | Responses API input item |
|---|---|
| `user` role, string content | `{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "..."}]}` |
| `user` role, `{"type": "text"}` block | `{"type": "input_text", "text": "..."}` inside a user message |
| `user` role, `{"type": "image", "source": {"type": "base64"}}` | `{"type": "input_image", "image_url": "data:<media_type>;base64,<data>"}` inside a user message |
| `user` role, `{"type": "image", "source": {"type": "url"}}` | `{"type": "input_image", "image_url": "<url>"}` inside a user message |
| `user` role, `{"type": "tool_result"}` block | Top-level `{"type": "function_call_output", "call_id": "...", "output": "..."}` — pulled out of the message entirely |
| `assistant` role, string content | `{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "..."}]}` |
| `assistant` role, `{"type": "text"}` block | `{"type": "output_text", "text": "..."}` inside an assistant message |
| `assistant` role, `{"type": "tool_use"}` block | Top-level `{"type": "function_call", "call_id": "<id>", "name": "...", "arguments": "<JSON string>"}` — pulled out of the message entirely |
| `assistant` role, `{"type": "thinking"}` block | `{"type": "output_text", "text": "<thinking text>"}` inside an assistant message |
### tools
| Anthropic tool | Responses API tool |
|---|---|
| Any tool where `type` starts with `"web_search"` or `name == "web_search"` | `{"type": "web_search_preview"}` |
| All other tools | `{"type": "function", "name": "...", "description": "...", "parameters": <input_schema>}` |
### tool_choice
| Anthropic `tool_choice.type` | Responses API `tool_choice` |
|---|---|
| `"auto"` | `{"type": "auto"}` |
| `"any"` | `{"type": "required"}` |
| `"tool"` | `{"type": "function", "name": "<tool name>"}` |
### thinking → reasoning
The `budget_tokens` value is mapped to a string effort level. `summary` is always set to `"detailed"`.
| `thinking.budget_tokens` | `reasoning.effort` |
|---|---|
| >= 10000 | `"high"` |
| >= 5000 | `"medium"` |
| >= 2000 | `"low"` |
| < 2000 | `"minimal"` |
If `thinking.type` is anything other than `"enabled"`, the `reasoning` field is not sent at all.
### context_management
Anthropic uses a nested dict with an `edits` array. OpenAI uses a flat array of compaction objects.
```
Anthropic input:
{
"edits": [
{
"type": "compact_20260112",
"trigger": {"type": "input_tokens", "value": 150000}
}
]
}
Responses API output:
[
{"type": "compaction", "compact_threshold": 150000}
]
```
## Response: Responses API → Anthropic
When the Responses API reply comes back, LiteLLM converts it into an Anthropic `AnthropicMessagesResponse`.
| Responses API field | Anthropic response field | Notes |
|---|---|---|
| `response.id` | `id` | |
| `response.model` | `model` | Falls back to `"unknown-model"` if missing |
| `ResponseReasoningItem` — `summary[*].text` | `content` block `{"type": "thinking", "thinking": "..."}` | Each non-empty summary text becomes a thinking block |
| `ResponseOutputMessage` — `content[*]` where `type == "output_text"` | `content` block `{"type": "text", "text": "..."}` | |
| `ResponseFunctionToolCall` — `{call_id, name, arguments}` | `content` block `{"type": "tool_use", "id": "...", "name": "...", "input": {...}}` | `arguments` is JSON-parsed back into a dict |
| Any `function_call` present in output | `stop_reason: "tool_use"` | |
| `response.status == "incomplete"` | `stop_reason: "max_tokens"` | Takes precedence over the default |
| Everything else | `stop_reason: "end_turn"` | Default |
| `response.usage.input_tokens` | `usage.input_tokens` | |
| `response.usage.output_tokens` | `usage.output_tokens` | |
| *(hardcoded)* | `type: "message"` | Always set |
| *(hardcoded)* | `role: "assistant"` | Always set |
| *(hardcoded)* | `stop_sequence: null` | Always null on this path |

1
docs/my-website/docs/apply_guardrail.md
View file

@ -11,6 +11,7 @@ This endpoint supports various guardrail types including:
- **Presidio** - PII detection and masking
- **Bedrock** - AWS Bedrock guardrails for content moderation
- **Lakera** - AI safety guardrails
- **PANW Prisma AIRS** - Threat detection, DLP, and policy enforcement
- **Custom guardrails** - User-defined guardrails
## Configuration

3
docs/my-website/docs/audio_transcription.md
View file

@ -13,7 +13,7 @@ import TabItem from '@theme/TabItem';
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to output transcribed text (non-streaming only) |
| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud` | |
| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud`, `mistral` | |
## Quick Start
@ -126,6 +126,7 @@ transcript = client.audio.transcriptions.create(
- [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
- [Groq](./providers/groq.md#speech-to-text---whisper)
- [Deepgram](./providers/deepgram.md)
- [Mistral (Voxtral)](./providers/mistral.md#audio-transcription)
- [OVHcloud AI Endpoints](./providers/ovhcloud.md)
---

38
docs/my-website/docs/benchmarks.md
View file

@ -5,6 +5,44 @@ import Image from '@theme/IdealImage';
Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpoint.
## Setting Up Benchmarking with Network Mock
The fastest way to benchmark proxy overhead is using `network_mock` mode. This intercepts outbound requests at the httpx transport layer and returns canned responses, no need for setting up a mock provider.
**1. Create a proxy config:**
```yaml
model_list:
- model_name: db-openai-endpoint
litellm_params:
model: openai/gpt-4o
api_key: "sk-fake-key"
api_base: "https://api.openai.com"
litellm_settings:
network_mock: true
callbacks: []
num_retries: 0
request_timeout: 30
general_settings:
master_key: "sk-1234"
```
**2. Start the proxy:**
```bash
litellm --config benchmark_config.yaml --port 4000 --num_workers 8
```
**3. Run the benchmark script:**
```bash
python scripts/benchmark_mock.py --requests 2000 --max-concurrent 200 --runs 3
```
This measures pure proxy overhead on the hot path without any network latency to a real or fake provider.
## Setting Up a Fake OpenAI Endpoint
For load testing and benchmarking, you can use a fake OpenAI proxy server. LiteLLM provides:

2
docs/my-website/docs/caching/all_caches.md
View file

@ -297,6 +297,7 @@ litellm.cache = Cache(
similarity_threshold=0.7, # similarity threshold for cache hits, 0 == no similarity, 1 = exact matches, 0.5 == 50% similarity
qdrant_quantization_config ="binary", # can be one of 'binary', 'product' or 'scalar' quantizations that is supported by qdrant
qdrant_semantic_cache_embedding_model="text-embedding-ada-002", # this model is passed to litellm.embedding(), any litellm.embedding() model is supported here
qdrant_semantic_cache_vector_size=1536, # vector size for the embedding model, must match the dimensionality of the embedding model used
)
response1 = completion(
@ -635,6 +636,7 @@ def __init__(
qdrant_quantization_config: Optional[str] = None,
qdrant_semantic_cache_embedding_model="text-embedding-ada-002",
qdrant_semantic_cache_vector_size: Optional[int] = None,
**kwargs
):
```

22
docs/my-website/docs/completion/output.md
View file

@ -51,6 +51,28 @@ Here's what an example response looks like
}
```
## Native Finish Reason
LiteLLM maps all provider-specific `finish_reason` values to OpenAI-compatible values (`stop`, `length`, `tool_calls`, `function_call`, `content_filter`). When the original provider value differs from the mapped value, it is preserved in `provider_specific_fields["native_finish_reason"]`.
This is useful for agent loops that need to distinguish between different stop conditions (e.g., Gemini's `MALFORMED_FUNCTION_CALL` vs a normal `stop`).
```python
response = completion(model="gemini/gemini-2.0-flash", messages=messages)
choice = response.choices[0]
print(choice.finish_reason) # "stop" (OpenAI-compatible)
# Access the original provider value when it differs:
if hasattr(choice, "provider_specific_fields") and choice.provider_specific_fields:
native = choice.provider_specific_fields.get("native_finish_reason")
if native == "MALFORMED_FUNCTION_CALL":
# Handle malformed function call differently from a normal stop
pass
```
When the provider already returns an OpenAI-compatible value (e.g., `stop`), `native_finish_reason` is not set.
## Additional Attributes
You can also access information like latency.

82
docs/my-website/docs/completion/prompt_caching.md
View file

@ -63,7 +63,6 @@ for _ in range(2):
}
],
},
# marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
{
"role": "user",
"content": [
@ -77,7 +76,6 @@ for _ in range(2):
"role": "assistant",
"content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
},
# The final turn is marked with cache-control, for continuing in followups.
{
"role": "user",
"content": [
@ -112,16 +110,16 @@ model_list:
api_key: os.environ/OPENAI_API_KEY
```
2. Start proxy
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
3. Test it!
```python
from openai import OpenAI
from openai import OpenAI
import os
client = OpenAI(
@ -144,7 +142,6 @@ for _ in range(2):
}
],
},
# marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
{
"role": "user",
"content": [
@ -158,7 +155,6 @@ for _ in range(2):
"role": "assistant",
"content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
},
# The final turn is marked with cache-control, for continuing in followups.
{
"role": "user",
"content": [
@ -183,6 +179,78 @@ assert response.usage.prompt_tokens_details.cached_tokens > 0
</TabItem>
</Tabs>
### OpenAI `prompt_cache_key` and `prompt_cache_retention`
OpenAI prompt caching is [**automatic**](https://platform.openai.com/docs/guides/prompt-caching) — no `cache_control` message annotations are needed. Any request with 1024+ prompt tokens is eligible for caching.
OpenAI also supports two optional parameters for more control over caching behavior:
- **`prompt_cache_key`** (string) — A routing hint that improves cache hit rates for requests sharing long common prefixes. Requests with the same cache key are routed to the same backend, increasing the likelihood of a cache hit.
- **`prompt_cache_retention`** (`"in_memory"` or `"24h"`) — Controls cache TTL. Default is `"in_memory"` (5–10 min). Set to `"24h"` for extended caching that offloads KV tensors to GPU-local storage.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ["OPENAI_API_KEY"] = ""
response = completion(
model="gpt-4o",
messages=[
{
"role": "system",
"content": "You are an AI assistant tasked with analyzing legal documents. "
+ "Here is the full text of a complex legal agreement " * 400,
},
{
"role": "user",
"content": "What are the key terms and conditions?",
},
],
prompt_cache_key="legal-doc-analysis",
prompt_cache_retention="24h",
)
print(response.usage)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```python
from openai import OpenAI
client = OpenAI(
api_key="LITELLM_PROXY_KEY",
base_url="LITELLM_PROXY_BASE",
)
response = client.chat.completions.create(
model="gpt-4o",
messages=[
{
"role": "system",
"content": "You are an AI assistant tasked with analyzing legal documents. "
+ "Here is the full text of a complex legal agreement " * 400,
},
{
"role": "user",
"content": "What are the key terms and conditions?",
},
],
extra_body={
"prompt_cache_key": "legal-doc-analysis",
"prompt_cache_retention": "24h",
},
)
print(response.usage)
```
</TabItem>
</Tabs>
### Anthropic Example
Anthropic charges for cache writes.

5
docs/my-website/docs/completion/web_fetch.md
View file

@ -115,6 +115,11 @@ print(response)
Web fetch is available on the following Anthropic API models:
- `claude-opus-4-6` (Claude Opus 4.6)
- `claude-sonnet-4-6` (Claude Sonnet 4.6)
- `claude-opus-4-5` (Claude Opus 4.5)
- `claude-sonnet-4-5` (Claude Sonnet 4.5)
- `claude-haiku-4-5` (Claude Haiku 4.5)
- `claude-opus-4-1-20250805` (Claude Opus 4.1)
- `claude-opus-4-20250514` (Claude Opus 4)
- `claude-sonnet-4-20250514` (Claude Sonnet 4)

22
docs/my-website/docs/contributing.md
View file

@ -79,7 +79,27 @@ cp -r out/* ../../litellm/proxy/_experimental/out/
Then restart the proxy and access the UI at `http://localhost:4000/ui`
## 4. Submitting a PR
## 4. Pre-PR Checklist
Before submitting your pull request, make sure the following pass locally from `ui/litellm-dashboard/`:
**Run tests related to your changes:**
```bash
npx vitest run src/components/path/to/YourComponent.test.tsx
```
Tests are co-located with components (e.g., `TeamInfo.tsx` → `TeamInfo.test.tsx`). If you add a new component, add a corresponding `.test.tsx` file next to it.
**Run the build:**
```bash
npm run build
```
These map to the `ui_tests` and `ui_build` CI checks.
## 5. Submitting a PR
1. Create a new branch for your changes:
```bash

42
docs/my-website/docs/contributing/adding_openai_compatible_providers.md
View file

@ -80,6 +80,36 @@ That's it! The provider is now available.
}
```
## Responses API Support
If your provider also supports the OpenAI Responses API (`/v1/responses`), add `supported_endpoints`:
```json
{
"your_provider": {
"base_url": "https://api.yourprovider.com/v1",
"api_key_env": "YOUR_PROVIDER_API_KEY",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
}
}
```
This enables `litellm.responses()` with zero additional code:
```python
import litellm
response = litellm.responses(
model="your_provider/model-name",
input="Hello, what can you do?",
)
print(response.output)
```
If `supported_endpoints` is omitted, it defaults to `[]`. Chat completions is always enabled for JSON providers regardless of this field.
The provider inherits all request/response handling from OpenAI's Responses API — streaming, tools, and all standard parameters work out of the box.
## Usage
```python
@ -89,11 +119,17 @@ import os
# Set your API key
os.environ["YOUR_PROVIDER_API_KEY"] = "your-key-here"
# Use the provider
# Chat completions
response = litellm.completion(
model="your_provider/model-name",
messages=[{"role": "user", "content": "Hello"}],
)
# Responses API (if supported_endpoints includes "/v1/responses")
response = litellm.responses(
model="your_provider/model-name",
input="Hello",
)
```
## When to Use Python Instead
@ -105,7 +141,9 @@ Use a Python config class if you need:
- Provider-specific streaming logic
- Advanced tool calling modifications
For these cases, create a config class in `litellm/llms/your_provider/chat/transformation.py` that inherits from `OpenAIGPTConfig` or `OpenAILikeChatConfig`.
For chat completions, create a config class in `litellm/llms/your_provider/chat/transformation.py` that inherits from `OpenAIGPTConfig` or `OpenAILikeChatConfig`.
For responses API with small overrides, inherit from `OpenAIResponsesAPIConfig` and override only what's needed. See `litellm/llms/perplexity/responses/transformation.py` for a minimal example (~40 lines vs 400+).
## Testing

189
docs/my-website/docs/count_tokens.md Normal file
View file

@ -0,0 +1,189 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Token Counting
## Overview
LiteLLM provides exact token counting by calling provider-specific token counting APIs. This gives you accurate token counts before sending requests, helping with cost estimation and context window management.
| Feature | Details |
|---------|---------|
| SDK Method | `litellm.acount_tokens()` |
| Proxy Endpoints | `/v1/messages/count_tokens` (Anthropic format), `/v1/responses/input_tokens` (OpenAI format) |
| Fallback | Local tiktoken-based counting for unsupported providers |
## Supported Providers
| Provider | Token Counting API | Format |
|----------|-------------------|--------|
| OpenAI | [Responses API `/input_tokens`](https://platform.openai.com/docs/api-reference/responses/input-tokens) | OpenAI Responses |
| Anthropic | [Messages `/count_tokens`](https://docs.anthropic.com/en/docs/build-with-claude/token-counting) | Anthropic Messages |
| Vertex AI (Claude) | Vertex AI Partner Models Token Counter | Anthropic Messages |
| Bedrock (Claude) | AWS Bedrock CountTokens API | Anthropic Messages |
| Gemini | Google AI Studio countTokens API | Anthropic Messages |
| Vertex AI (Gemini) | Vertex AI countTokens API | Anthropic Messages |
| Other providers | Local tiktoken fallback | N/A |
## SDK Usage
### Basic Usage
```python
import asyncio
import litellm
async def main():
# OpenAI
result = await litellm.acount_tokens(
model="openai/gpt-4o",
messages=[{"role": "user", "content": "Hello, how are you?"}],
)
print(f"Token count: {result.total_tokens}")
print(f"Tokenizer: {result.tokenizer_type}") # "openai_api"
# Anthropic
result = await litellm.acount_tokens(
model="anthropic/claude-3-5-sonnet-20241022",
messages=[{"role": "user", "content": "Hello, how are you?"}],
)
print(f"Token count: {result.total_tokens}")
print(f"Tokenizer: {result.tokenizer_type}") # "anthropic_api"
asyncio.run(main())
```
### With Tools and System Message
```python
import asyncio
import litellm
async def main():
result = await litellm.acount_tokens(
model="openai/gpt-4o",
messages=[{"role": "user", "content": "What's the weather in Paris?"}],
tools=[{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather for a city",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}],
system="You are a helpful weather assistant.",
)
print(f"Token count (with tools): {result.total_tokens}")
asyncio.run(main())
```
### Response Format
`litellm.acount_tokens()` returns a `TokenCountResponse`:
```python
TokenCountResponse(
total_tokens=15, # Token count
request_model="openai/gpt-4o", # Model requested
model_used="gpt-4o", # Model used for counting
tokenizer_type="openai_api", # "openai_api", "anthropic_api", "local_tokenizer"
original_response={"input_tokens": 15}, # Raw API response
error=False, # True if counting failed
error_message=None, # Error details if failed
)
```
### Fallback Behavior
If a provider doesn't support a token counting API, or if the API key is missing, `acount_tokens()` automatically falls back to local tiktoken-based counting:
```python
# Unsupported provider → automatic fallback
result = await litellm.acount_tokens(
model="together_ai/meta-llama/Llama-3-8b-chat-hf",
messages=[{"role": "user", "content": "Hello"}],
)
print(result.tokenizer_type) # "local_tokenizer"
```
## Proxy Usage
### OpenAI Format — `/v1/responses/input_tokens`
<Tabs>
<TabItem value="curl" label="curl">
```bash
curl -X POST "http://localhost:4000/v1/responses/input_tokens" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4o",
"input": "Hello, how are you?"
}'
```
</TabItem>
<TabItem value="python" label="Python (httpx)">
```python
import httpx
response = httpx.post(
"http://localhost:4000/v1/responses/input_tokens",
headers={
"Content-Type": "application/json",
"Authorization": "Bearer sk-1234"
},
json={
"model": "gpt-4o",
"input": "Hello, how are you?"
}
)
print(response.json())
# {"input_tokens": 7}
```
</TabItem>
</Tabs>
**Response:**
```json
{"input_tokens": 7}
```
### Anthropic Format — `/v1/messages/count_tokens`
See [Anthropic Token Counting](./anthropic_count_tokens.md) for full documentation.
```bash
curl -X POST "http://localhost:4000/v1/messages/count_tokens" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "claude-3-5-sonnet-20241022",
"messages": [
{"role": "user", "content": "Hello, how are you?"}
]
}'
```
## Proxy Configuration
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
- model_name: claude-3-5-sonnet
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
```

51
docs/my-website/docs/embedding/supported_embedding.md
View file

@ -514,6 +514,57 @@ All models listed [here](https://ai.google.dev/gemini-api/docs/models/gemini) ar
| Model Name | Function Call |
| :--- | :--- |
| text-embedding-004 | `embedding(model="gemini/text-embedding-004", input)` |
| gemini-embedding-2-preview | `embedding(model="gemini/gemini-embedding-2-preview", input)` | [Multimodal docs](#gemini-embedding-2-preview-multimodal) |
### Gemini Embedding 2 Preview (Multimodal)
`gemini-embedding-2-preview` supports **multimodal embeddings**—text, images, audio, video, and PDF in a single request. See [blog post](/blog/gemini_embedding_2_multimodal) for details.
**Input formats:**
- **Data URIs:** `data:image/png;base64,<encoded_data>`
- **Gemini file references:** `files/abc123` (pre-uploaded via Gemini Files API)
**Supported MIME types:** `image/png`, `image/jpeg`, `audio/mpeg`, `audio/wav`, `video/mp4`, `video/quicktime`, `application/pdf`
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import embedding
import os
os.environ["GEMINI_API_KEY"] = ""
# Text + Image (base64)
response = embedding(
model="gemini/gemini-embedding-2-preview",
input=[
"The food was delicious and the waiter...",
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
],
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl -X POST http://localhost:4000/embeddings \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gemini-embedding-2-preview",
"input": [
"The food was delicious and the waiter...",
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
]
}'
```
</TabItem>
</Tabs>
**Optional:** `dimensions` maps to Gemini's `outputDimensionality`.
## Vertex AI Embedding Models

6
docs/my-website/docs/enterprise.md
View file

@ -4,7 +4,7 @@ import Image from '@theme/IdealImage';
:::info
- ✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) to discuss your needs.
- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) to discuss your needs.
:::
For companies that need SSO, user management and professional support for LiteLLM Proxy
@ -36,7 +36,7 @@ Manage Yourself - you can deploy our Docker Image or build a custom image from o
### What’s the cost of the Self-Managed Enterprise edition?
Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
### How does deployment with Enterprise License work?
@ -106,7 +106,7 @@ Professional Support can assist with LLM/Provider integrations, deployment, upgr
Pricing is based on usage. We can figure out a price that works for your team, on the call.
[**Contact Us to learn more**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[**Contact Us to learn more**](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)

2
docs/my-website/docs/fine_tuning.md
View file

@ -6,7 +6,7 @@ import TabItem from '@theme/TabItem';
:::info
This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

1
docs/my-website/docs/generateContent.md
View file

@ -15,6 +15,7 @@ Use LiteLLM to call Google AI's generateContent endpoints for text generation, m
| Streaming | ✅ | |
| Fallbacks | ✅ | between supported models |
| Loadbalancing | ✅ | between supported models |
| Metadata Tracking | ✅ | passes trace ID, metadata to observability callbacks (e.g. S3, Langfuse) |
## Usage
---

157
docs/my-website/docs/image_edits.md
View file

@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| Supported operations | Create image edits | Single and multiple images supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
| Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **Stability AI**, **AWS Bedrock (Stability)** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. Stability AI and Bedrock Stability support various image editing operations. |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **OpenRouter**, **Stability AI**, **AWS Bedrock (Stability)**, **Black Forest Labs** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. OpenRouter routes image edits through chat completions. Stability AI and Bedrock Stability support various image editing operations. Black Forest Labs supports FLUX Kontext models. |
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
@ -199,6 +199,63 @@ for idx, image_obj in enumerate(response.data):
</TabItem>
<TabItem value="bfl" label="Black Forest Labs">
#### Basic Image Edit
```python showLineNumbers title="Black Forest Labs Image Edit"
import os
import litellm
os.environ["BFL_API_KEY"] = "your-api-key"
response = litellm.image_edit(
model="black_forest_labs/flux-kontext-pro",
image=open("original_image.png", "rb"),
prompt="Add a green leaf to the scene",
)
print(response.data[0].url)
```
#### Inpainting with Mask
```python showLineNumbers title="Black Forest Labs Inpainting"
import os
import litellm
os.environ["BFL_API_KEY"] = "your-api-key"
# Use flux-pro-1.0-fill for inpainting
response = litellm.image_edit(
model="black_forest_labs/flux-pro-1.0-fill",
image=open("original_image.png", "rb"),
mask=open("mask_image.png", "rb"),
prompt="Replace with a garden",
)
print(response.data[0].url)
```
#### Outpainting (Expand)
```python showLineNumbers title="Black Forest Labs Outpainting"
import os
import litellm
os.environ["BFL_API_KEY"] = "your-api-key"
# Use flux-pro-1.0-expand to extend image borders
response = litellm.image_edit(
model="black_forest_labs/flux-pro-1.0-expand",
image=open("original_image.png", "rb"),
prompt="Continue the scene with mountains",
top=256,
bottom=256,
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="vertex_ai" label="Vertex AI">
#### Basic Image Edit (Gemini)
@ -244,6 +301,47 @@ response = litellm.image_edit(
print(response)
```
</TabItem>
<TabItem value="openrouter" label="OpenRouter">
#### Basic Image Edit
```python showLineNumbers title="OpenRouter Image Edit"
import os
from litellm import image_edit
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=open("original_image.png", "rb"),
prompt="Add aurora borealis to the night sky",
)
print(response)
```
#### Multiple Images Edit
```python showLineNumbers title="OpenRouter Multiple Images Edit"
import os
from litellm import image_edit
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=[
open("scene.png", "rb"),
open("style_reference.png", "rb"),
],
prompt="Blend the reference style into the scene",
size="1536x1024", # mapped to aspect_ratio 3:2
quality="high", # mapped to image_size 4K
)
print(response)
```
</TabItem>
</Tabs>
@ -351,6 +449,35 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
</TabItem>
<TabItem value="bfl" label="Black Forest Labs">
1. Add Black Forest Labs image edit models to your `config.yaml`:
```yaml showLineNumbers title="Black Forest Labs Proxy Configuration"
model_list:
- model_name: bfl-kontext-pro
litellm_params:
model: black_forest_labs/flux-kontext-pro
api_key: os.environ/BFL_API_KEY
model_info:
mode: image_edit
```
2. Start the LiteLLM proxy server:
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
```
3. Make an image edit request:
```bash showLineNumbers title="Black Forest Labs Proxy Image Edit"
curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-F "model=bfl-kontext-pro" \
-F "image=@original_image.png" \
-F "prompt=Add a sunset in the background"
```
</TabItem>
<TabItem value="vertex_ai" label="Vertex AI">
1. Add Vertex AI image edit models to your `config.yaml`:
@ -398,6 +525,34 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-F "size=1024x1024"
```
</TabItem>
<TabItem value="openrouter" label="OpenRouter">
1. Add the OpenRouter image edit model to your `config.yaml`:
```yaml showLineNumbers title="OpenRouter Proxy Configuration"
model_list:
- model_name: openrouter-image-edit
litellm_params:
model: openrouter/google/gemini-2.5-flash-image
api_key: os.environ/OPENROUTER_API_KEY
```
2. Start the LiteLLM proxy server:
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
```
3. Make an image edit request:
```bash showLineNumbers title="OpenRouter Proxy Image Edit"
curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-F "model=openrouter-image-edit" \
-F "image=@original_image.png" \
-F "prompt=Make the sky a vibrant purple sunset" \
-F "size=1024x1024"
```
</TabItem>
</Tabs>

2
docs/my-website/docs/image_generation.md
View file

@ -15,7 +15,7 @@ import TabItem from '@theme/TabItem';
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input prompts (non-streaming only) |
| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, OpenRouter, Xinference, Nscale | |
| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Black Forest Labs, Recraft, OpenRouter, Xinference, Nscale | |
## Quick Start

18
docs/my-website/docs/interactions.md
View file

@ -130,13 +130,12 @@ Point the Google GenAI SDK to LiteLLM Proxy:
```python showLineNumbers title="Google GenAI SDK with LiteLLM Proxy"
from google import genai
import os
# Point SDK to LiteLLM Proxy
os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
os.environ["GEMINI_API_KEY"] = "sk-1234" # Your LiteLLM API key
client = genai.Client()
client = genai.Client(
api_key="sk-1234", # Your LiteLLM API key
http_options={"base_url": "http://localhost:4000"},
)
# Create an interaction
interaction = client.interactions.create(
@ -151,12 +150,11 @@ print(interaction.outputs[-1].text)
```python showLineNumbers title="Google GenAI SDK Streaming"
from google import genai
import os
os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
os.environ["GEMINI_API_KEY"] = "sk-1234"
client = genai.Client()
client = genai.Client(
api_key="sk-1234", # Your LiteLLM API key
http_options={"base_url": "http://localhost:4000"},
)
for chunk in client.interactions.create_stream(
model="gemini/gemini-2.5-flash",

255
docs/my-website/docs/mcp.md
View file

@ -133,6 +133,21 @@ LiteLLM attempts [OAuth 2.0 Authorization Server Discovery](https://datatracker.
<br/>
### AWS SigV4 Authentication
For MCP servers hosted on [AWS Bedrock AgentCore](https://docs.aws.amazon.com/bedrock/latest/userguide/agentcore.html), select **AWS SigV4** as the authentication type. LiteLLM will sign every outgoing MCP request with your AWS credentials using [Signature Version 4](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html).
<Image
img={require('../img/mcp_aws_sigv4_ui.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
Fill in your AWS region, service name (defaults to `bedrock-agentcore`), and optionally your AWS access key and secret. If credentials are omitted, LiteLLM falls back to the boto3 credential chain (IAM roles, environment variables, etc.).
[**See full SigV4 setup guide**](./mcp_aws_sigv4.md)
<br/>
### Static Headers
Sometimes your MCP server needs specific headers on every request. Maybe it's an API key, maybe it's a custom header the server expects. Instead of configuring auth, you can just set them directly.
@ -217,6 +232,7 @@ mcp_servers:
| `bearer_token` | `Authorization: Bearer <auth_value>` |
| `basic` | `Authorization: Basic <auth_value>` |
| `authorization` | `Authorization: <auth_value>` |
| `aws_sigv4` | Per-request AWS SigV4 signature ([details](./mcp_aws_sigv4.md)) |
- **Extra Headers**: Optional list of additional header names that should be forwarded from client to the MCP server
- **Static Headers**: Optional map of header key/value pairs to include every request to the MCP server.
@ -257,6 +273,16 @@ mcp_servers:
auth_type: "authorization"
auth_value: "Token example123" # headers={"Authorization": "Token example123"}
# AWS SigV4 for Bedrock AgentCore MCP servers
agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
aws_service_name: bedrock-agentcore
# Example with extra headers forwarding
github_mcp:
url: "https://api.githubcopilot.com/mcp"
@ -336,175 +362,9 @@ litellm_settings:
## Converting OpenAPI Specs to MCP Servers
LiteLLM can automatically convert OpenAPI specifications into MCP servers, allowing you to expose any REST API as MCP tools. This is useful when you have existing APIs with OpenAPI/Swagger documentation and want to make them available as MCP tools.
LiteLLM can convert OpenAPI specifications into MCP servers, exposing any REST API as MCP tools without writing custom server code.
**Benefits:**
- **Rapid Integration**: Convert existing APIs to MCP tools without writing custom MCP server code
- **Automatic Tool Generation**: LiteLLM automatically generates MCP tools from your OpenAPI spec
- **Unified Interface**: Use the same MCP interface for both native MCP servers and OpenAPI-based APIs
- **Easy Testing**: Test and iterate on API integrations quickly
**Configuration:**
Add your OpenAPI-based MCP server to your `config.yaml`:
```yaml title="config.yaml - OpenAPI to MCP" showLineNumbers
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-xxxxxxx
mcp_servers:
# OpenAPI Spec Example - Petstore API
petstore_mcp:
url: "https://petstore.swagger.io/v2"
spec_path: "/path/to/openapi.json"
auth_type: "none"
# OpenAPI Spec with API Key Authentication
my_api_mcp:
url: "http://0.0.0.0:8090"
spec_path: "/path/to/openapi.json"
auth_type: "api_key"
auth_value: "your-api-key-here"
# OpenAPI Spec with Bearer Token
secured_api_mcp:
url: "https://api.example.com"
spec_path: "/path/to/openapi.json"
auth_type: "bearer_token"
auth_value: "your-bearer-token"
```
**Configuration Parameters:**
| Parameter | Required | Description |
|-----------|----------|-------------|
| `url` | Yes | The base URL of your API endpoint |
| `spec_path` | Yes | Path or URL to your OpenAPI specification file (JSON or YAML) |
| `auth_type` | No | Authentication type: `none`, `api_key`, `bearer_token`, `basic`, `authorization` |
| `auth_value` | No | Authentication value (required if `auth_type` is set) |
| `authorization_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `token_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `registration_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `scopes` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM uses the scopes advertised by the server. |
| `description` | No | Optional description for the MCP server |
| `allowed_tools` | No | List of specific tools to allow (see [MCP Tool Filtering](#mcp-tool-filtering)) |
| `disallowed_tools` | No | List of specific tools to block (see [MCP Tool Filtering](#mcp-tool-filtering)) |
### Usage Example
Once configured, you can use the OpenAPI-based MCP server just like any other MCP server:
<Tabs>
<TabItem value="fastmcp" label="Python FastMCP">
```python title="Using OpenAPI-based MCP Server" showLineNumbers
from fastmcp import Client
import asyncio
# Standard MCP configuration
config = {
"mcpServers": {
"petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
}
}
}
# Create a client that connects to the server
client = Client(config)
async def main():
async with client:
# List available tools generated from OpenAPI spec
tools = await client.list_tools()
print(f"Available tools: {[tool.name for tool in tools]}")
# Example: Get a pet by ID (from Petstore API)
response = await client.call_tool(
name="getpetbyid",
arguments={"petId": "1"}
)
print(f"Response:\n{response}\n")
# Example: Find pets by status
response = await client.call_tool(
name="findpetsbystatus",
arguments={"status": "available"}
)
print(f"Response:\n{response}\n")
if __name__ == "__main__":
asyncio.run(main())
```
</TabItem>
<TabItem value="cursor" label="Cursor IDE">
```json title="Cursor MCP Configuration for OpenAPI Server" showLineNumbers
{
"mcpServers": {
"Petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY"
}
}
}
}
```
</TabItem>
<TabItem value="openai" label="OpenAI Responses API">
```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
curl --location 'https://api.openai.com/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer $OPENAI_API_KEY" \
--data '{
"model": "gpt-4o",
"tools": [
{
"type": "mcp",
"server_label": "petstore",
"server_url": "http://localhost:4000/petstore_mcp/mcp",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
}
}
],
"input": "Find all available pets in the petstore",
"tool_choice": "required"
}'
```
</TabItem>
</Tabs>
**How It Works**
1. **Spec Loading**: LiteLLM loads your OpenAPI specification from the provided `spec_path`
2. **Tool Generation**: Each API endpoint in the spec becomes an MCP tool
3. **Parameter Mapping**: OpenAPI parameters are automatically mapped to MCP tool parameters
4. **Request Handling**: When a tool is called, LiteLLM converts the MCP request to the appropriate HTTP request
5. **Response Translation**: API responses are converted back to MCP format
**OpenAPI Spec Requirements**
Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
- **Supported versions**: OpenAPI 3.0.x, OpenAPI 3.1.x, Swagger 2.0
- **Required fields**: `paths`, `info` sections should be properly defined
- **Operation IDs**: Each operation should have a unique `operationId` (this becomes the tool name)
- **Parameters**: Request parameters should be properly documented with types and descriptions
See the **[MCP from OpenAPI Specs guide](./mcp_openapi.md)** for full setup, usage examples, and how to override tool names and descriptions.
## MCP OAuth
@ -641,7 +501,7 @@ import asyncio
config = {
"mcpServers": {
"mcp_group": {
"url": "http://localhost:4000/mcp",
"url": "http://localhost:4000/mcp/",
"headers": {
"x-mcp-servers": "dev_group", # assume this gives access to github, zapier and deepwiki
"x-litellm-api-key": "Bearer sk-1234",
@ -870,6 +730,63 @@ asyncio.run(main())
[Learn more about customer management →](./proxy/customers)
## Calling the Proxy's /v1/responses Endpoint
When calling your LiteLLM Proxy's `/v1/responses` endpoint to use MCP tools, **always use `server_url: "litellm_proxy"`** in the tools array. This tells the proxy to use its configured MCP servers.
:::important Do not use the full proxy URL
Using `server_url: "https://your-proxy.com/mcp"` is incorrect when the request is already going to the proxy. The proxy needs the literal value `litellm_proxy` to route to its configured MCP servers.
:::
```bash title="Correct: Using litellm_proxy" showLineNumbers
curl --location 'https://your-proxy.com/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer $LITELLM_API_KEY" \
--data '{
"model": "gpt-4",
"tools": [
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never"
}
],
"input": "Run available tools",
"tool_choice": "required"
}'
```
### Sending Custom Headers to MCP Servers
To pass custom headers (e.g., API keys, auth tokens) to specific MCP servers, use either:
**Option 1: Request headers** – Add `x-mcp-{server_alias}-{header_name}` to your request headers. The proxy forwards these to the matching MCP server.
```bash
# Send Authorization header to the "weather2" MCP server
--header 'x-mcp-weather2-authorization: Bearer your-token'
# Send custom header to the "github" MCP server
--header 'x-mcp-github-x-api-key: your-api-key'
```
**Option 2: Headers in tool config** – Include a `headers` object in the tool definition. These are merged with request headers.
```json
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
"x-mcp-servers": "Zapier_MCP,dev-group",
"x-mcp-weather2-authorization": "Bearer your-weather-api-token"
}
}
```
## Using your MCP with client side credentials
Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.

181
docs/my-website/docs/mcp_aws_sigv4.md Normal file
View file

@ -0,0 +1,181 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# MCP - AWS SigV4 Auth
Use AWS SigV4 authentication to connect LiteLLM to MCP servers hosted on [AWS Bedrock AgentCore](https://docs.aws.amazon.com/bedrock/latest/userguide/agentcore.html).
## Why SigV4?
AWS services authenticate requests using [Signature Version 4](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) — a per-request signing protocol that includes the request body in the cryptographic signature. This is fundamentally different from static-header auth types (`api_key`, `bearer_token`, etc.) which send the same header on every request.
LiteLLM's `aws_sigv4` auth type handles this automatically: every outgoing MCP request is signed with your AWS credentials before it's sent.
## Quick Start
<Tabs>
<TabItem value="ui" label="LiteLLM UI">
1. Navigate to **MCP Servers** and click **Add New MCP Server**
2. Set the transport to **Streamable HTTP**
3. Select **AWS SigV4** as the authentication type
4. Fill in your AWS credentials:
<Image
img={require('../img/mcp_aws_sigv4_ui.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
| Field | Required | Description |
|-------|----------|-------------|
| **AWS Region** | Yes | AWS region for SigV4 signing (e.g., `us-east-1`) |
| **AWS Service Name** | No | Defaults to `bedrock-agentcore` |
| **AWS Access Key ID** | No | Falls back to boto3 credential chain if blank |
| **AWS Secret Access Key** | No | Required if Access Key ID is provided |
| **AWS Session Token** | No | Only needed for temporary STS credentials |
Once created, LiteLLM will sign every outgoing MCP request with SigV4. The server's tools appear automatically in the MCP Tools list.
**Editing credentials:** When editing an existing SigV4 server, leave credential fields blank to keep the current values. Only fields you fill in will be updated.
</TabItem>
<TabItem value="config" label="config.yaml">
### 1. Set AWS credentials
```bash
export AWS_ACCESS_KEY_ID="AKIA..."
export AWS_SECRET_ACCESS_KEY="..."
export AWS_REGION_NAME="us-east-1"
```
### 2. Add your AgentCore MCP server to config.yaml
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
mcp_servers:
my_agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: "us-east-1"
aws_service_name: "bedrock-agentcore"
```
:::info URL encoding
The AgentCore runtime ARN must be URL-encoded in the `url` field. For example:
```
arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/my-mcp-server
```
becomes:
```
arn%3Aaws%3Abedrock-agentcore%3Aus-east-1%3A123456789012%3Aruntime%2Fmy-mcp-server
```
:::
### 3. Start the proxy
```bash
litellm --config config.yaml
```
</TabItem>
</Tabs>
## Use the MCP tools
Once configured, your AgentCore MCP tools are available through LiteLLM like any other MCP server:
```bash title="List available tools"
curl http://localhost:4000/mcp-rest/tools/list \
-H "Authorization: Bearer sk-1234"
```
```bash title="Call a tool"
curl http://localhost:4000/mcp-rest/tools/call \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"name": "my_agentcore_mcp_your_tool_name",
"arguments": {"key": "value"}
}'
```
## Config Reference
| Field | Required | Description |
|-------|----------|-------------|
| `url` | Yes | AgentCore MCP server URL (with URL-encoded ARN) |
| `transport` | Yes | Must be `"http"` |
| `auth_type` | Yes | Must be `"aws_sigv4"` |
| `aws_access_key_id` | No | AWS access key. Supports `os.environ/VAR_NAME`. Falls back to boto3 credential chain if omitted |
| `aws_secret_access_key` | No | AWS secret key. Supports `os.environ/VAR_NAME`. Falls back to boto3 credential chain if omitted |
| `aws_region_name` | Yes | AWS region (e.g., `us-east-1`) |
| `aws_service_name` | No | AWS service name for signing. Defaults to `bedrock-agentcore` |
| `aws_session_token` | No | AWS session token for temporary credentials. Supports `os.environ/VAR_NAME` |
## How It Works
LiteLLM uses an `httpx.Auth` subclass (`MCPSigV4Auth`) that hooks into the HTTP request lifecycle:
1. For every outgoing MCP request, the auth handler computes a SHA-256 hash of the request body
2. It creates a SigV4 signature using your AWS credentials, the request URL, headers, and body hash
3. The signed `Authorization` and `x-amz-date` headers are added to the request
4. AWS validates the signature and processes the MCP request
This happens transparently — no manual token management required.
## Using Temporary Credentials (STS)
If you use AWS STS temporary credentials (e.g., from IAM roles or SSO), include the session token:
```yaml title="config.yaml with STS credentials" showLineNumbers
mcp_servers:
my_agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_session_token: os.environ/AWS_SESSION_TOKEN
aws_region_name: "us-east-1"
aws_service_name: "bedrock-agentcore"
```
## Troubleshooting
### 403 Forbidden from AWS
- Verify your AWS credentials are valid and not expired
- Check that `aws_region_name` matches the region in your AgentCore URL
- Ensure `aws_service_name` is set to `bedrock-agentcore`
- If using STS credentials, confirm `aws_session_token` is set and not expired
### Health check errors on startup
SigV4-authenticated MCP servers skip the standard health check on proxy startup. This is expected — the proxy will still sign requests correctly when tools are invoked.
### "botocore not found" error
Install the `botocore` package:
```bash
pip install botocore
```
`botocore` is used for SigV4 credential handling and is required when using `aws_sigv4` auth.

8
docs/my-website/docs/mcp_control.md
View file

@ -323,7 +323,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
{
"type": "mcp",
"server_label": "litellm",
"server_url": "<your-litellm-proxy-base-url>/dev_group/mcp",
"server_url": "litellm_proxy",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
@ -335,7 +335,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
}'
```
This example uses URL namespacing to access all servers in the "dev_group" access group.
This example uses the `x-mcp-servers` header to access all servers in the "dev_group" access group. Use `server_url: "litellm_proxy"` when calling the proxy's `/v1/responses` endpoint—do not use the full proxy URL.
</TabItem>
@ -423,7 +423,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
{
"type": "mcp",
"server_label": "litellm",
"server_url": "<your-litellm-proxy-base-url>/mcp/",
"server_url": "litellm_proxy",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
@ -436,7 +436,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
}'
```
This configuration restricts the request to only use tools from the specified MCP servers.
This configuration restricts the request to only use tools from the specified MCP servers. Use `server_url: "litellm_proxy"` when calling the proxy's `/v1/responses` endpoint.
</TabItem>

1
docs/my-website/docs/mcp_guardrail.md
View file

@ -86,4 +86,5 @@ MCP guardrails work with all LiteLLM-supported guardrail providers:
- **Lakera**: Content moderation
- **Aporia**: Custom guardrails
- **Noma**: Noma Security
- **PANW Prisma AIRS**: Prisma AIRS guardrails
- **Custom**: Your own guardrail implementations

226
docs/my-website/docs/mcp_openapi.md Normal file
View file

@ -0,0 +1,226 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# MCP from OpenAPI Specs
LiteLLM can convert any OpenAPI/Swagger spec into an MCP server — no custom MCP server code required.
## Step 1 — Add the MCP Server
Add your OpenAPI-based server in `config.yaml`:
```yaml title="config.yaml" showLineNumbers
mcp_servers:
petstore_mcp:
url: "https://petstore.swagger.io/v2"
spec_path: "/path/to/openapi.json"
auth_type: "none"
my_api_mcp:
url: "http://0.0.0.0:8090"
spec_path: "/path/to/openapi.json"
auth_type: "api_key"
auth_value: "your-api-key-here"
secured_api_mcp:
url: "https://api.example.com"
spec_path: "/path/to/openapi.json"
auth_type: "bearer_token"
auth_value: "your-bearer-token"
```
Or from the UI: go to **MCP Servers → Add New MCP Server**, fill in the URL and spec path, and LiteLLM will fetch the spec and load all endpoints as tools.
**Configuration parameters:**
| Parameter | Required | Description |
|-----------|----------|-------------|
| `url` | Yes | Base URL of your API |
| `spec_path` | Yes | Path or URL to your OpenAPI spec (JSON or YAML) |
| `auth_type` | No | `none`, `api_key`, `bearer_token`, `basic`, `authorization`, `oauth2` |
| `auth_value` | No | Auth value (required if `auth_type` is set) |
| `description` | No | Optional description |
| `allowed_tools` | No | Allowlist of specific tools |
| `disallowed_tools` | No | Blocklist of specific tools |
**Supported spec versions:** OpenAPI 3.0.x, 3.1.x, Swagger 2.0. Each operation's `operationId` becomes the tool name — make sure they're unique.
Once tools are loaded, you'll see them in the Tool Configuration section:
<Image
img={require('../img/mcp_openapi_tools_loaded.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
## Step 2 — Optionally Override Tool Names and Descriptions
By default, tool names and descriptions come from the `operationId` and description fields in your spec. You can rename or rewrite them so MCP clients see something cleaner — without touching the upstream spec.
### From the UI
Each tool card has a pencil icon. Click it to open the inline editor:
<Image
img={require('../img/mcp_openapi_tool_edit_panel.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
- **Display Name** — overrides the name MCP clients see
- **Description** — overrides the description MCP clients see
- Leave a field blank to keep the original from the spec
After setting overrides, a purple **Custom name** badge appears on the tool card:
<Image
img={require('../img/mcp_openapi_custom_name_badge.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
### From the API
Pass `tool_name_to_display_name` and `tool_name_to_description` in the create or update request:
```bash title="Create server with tool name overrides" showLineNumbers
curl -X POST http://localhost:4000/v1/mcp/server \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"name": "petstore_mcp",
"url": "https://petstore.swagger.io/v2",
"spec_path": "/path/to/openapi.json",
"tool_name_to_display_name": {
"getPetById": "Get Pet",
"findPetsByStatus": "List Available Pets"
},
"tool_name_to_description": {
"getPetById": "Look up a pet by its ID",
"findPetsByStatus": "Returns all pets matching a given status (available, pending, sold)"
}
}'
```
```bash title="Update overrides on an existing server" showLineNumbers
curl -X PUT http://localhost:4000/v1/mcp/server/{server_id} \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"tool_name_to_display_name": {
"getPetById": "Get Pet"
},
"tool_name_to_description": {
"getPetById": "Look up a pet by its ID"
}
}'
```
The map key is the **original `operationId`** from the spec — not the prefixed tool name. LiteLLM strips the server prefix before doing the lookup.
For example, if your server is `petstore_mcp`, the tool is exposed as `petstore_mcp-getPetById`. The map key is still `getPetById`.
**Before and after:**
```
# Without overrides
Tool: "petstore_mcp-getPetById"
Description: "Returns a single pet"
Tool: "petstore_mcp-findPetsByStatus"
Description: "Finds Pets by status"
# After overrides
Tool: "Get Pet"
Description: "Look up a pet by its ID"
Tool: "List Available Pets"
Description: "Returns all pets matching a given status (available, pending, sold)"
```
## Using the Server
<Tabs>
<TabItem value="fastmcp" label="Python FastMCP">
```python title="Using OpenAPI-based MCP Server" showLineNumbers
from fastmcp import Client
import asyncio
config = {
"mcpServers": {
"petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
}
}
}
client = Client(config)
async def main():
async with client:
tools = await client.list_tools()
print(f"Available tools: {[tool.name for tool in tools]}")
response = await client.call_tool(
name="Get Pet", # overridden name
arguments={"petId": "1"}
)
print(f"Response: {response}")
if __name__ == "__main__":
asyncio.run(main())
```
</TabItem>
<TabItem value="cursor" label="Cursor IDE">
```json title="Cursor MCP Configuration" showLineNumbers
{
"mcpServers": {
"Petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY"
}
}
}
}
```
</TabItem>
<TabItem value="openai" label="OpenAI Responses API">
```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
curl --location 'https://api.openai.com/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer $OPENAI_API_KEY" \
--data '{
"model": "gpt-4o",
"tools": [
{
"type": "mcp",
"server_label": "petstore",
"server_url": "http://localhost:4000/petstore_mcp/mcp",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
}
}
],
"input": "Find all available pets",
"tool_choice": "required"
}'
```
</TabItem>
</Tabs>

60
docs/my-website/docs/observability/datadog.md
View file

@ -7,6 +7,7 @@ import TabItem from '@theme/TabItem';
LiteLLM Supports logging to the following Datdog Integrations:
- `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
- `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
- `datadog_metrics` [Datadog Custom Metrics](#datadog-custom-metrics)
- `datadog_cost_management` [Datadog Cloud Cost Management](#datadog-cloud-cost-management)
- `ddtrace-run` [Datadog Tracing](#datadog-tracing)
@ -168,6 +169,65 @@ On the Datadog LLM Observability page, you should see that both input messages a
<Image img={require('../../img/dd_llm_obs.png')} />
## Datadog Custom Metrics
| Feature | Details |
|---------|---------|
| **What is logged** | Latency metrics, request counts by status code |
| **Events** | Success + Failure |
| **Product Link** | [Datadog Metrics](https://docs.datadoghq.com/metrics/) |
Publishes the following metrics to Datadog via the `/api/v2/series` endpoint:
| Metric | Type | Description |
|--------|------|-------------|
| `litellm.request.total_latency` | Gauge | End-to-end request latency (seconds) |
| `litellm.llm_api.latency` | Gauge | Time spent waiting for the LLM provider response (seconds) |
| `litellm.llm_api.request_count` | Count | Request count, tagged with status code |
Using `total_latency` and `llm_api.latency`, you can derive **internal latency** = `total_latency - llm_api.latency`.
All metrics include the following tags: `env`, `service`, `version`, `HOSTNAME`, `POD_NAME`, `provider`, `model_name`, `model_group`, `team`, `status_code`.
**Step 1**: Create a `config.yaml` file
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
litellm_settings:
success_callback: ["datadog_metrics"]
failure_callback: ["datadog_metrics"]
```
**Step 2**: Set required env variables
```shell
DD_API_KEY="your-api-key"
DD_SITE="us5.datadoghq.com" # your datadog site
```
**Step 3**: Start the proxy and make a test request
```shell
litellm --config config.yaml
```
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "hello"}]
}'
```
**Step 4**: View metrics in Datadog Metrics Explorer
Navigate to **Metrics > Explorer** in Datadog and search for `litellm.request.total_latency`, `litellm.llm_api.latency`, or `litellm.llm_api.request_count`.
## Datadog Cloud Cost Management
| Feature | Details |

2
docs/my-website/docs/observability/gcs_bucket_integration.md
View file

@ -6,7 +6,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage?
:::info
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

98
docs/my-website/docs/ocr.md
View file

@ -61,6 +61,52 @@ async def test_async_ocr():
asyncio.run(test_async_ocr())
```
### Using Local Files
LiteLLM can read local files directly — no manual base64 encoding needed:
```python
from litellm import ocr
# OCR with a local PDF file path
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "file",
"file": "/path/to/document.pdf"
}
)
# OCR with a file object
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "file",
"file": open("document.pdf", "rb")
}
)
# OCR with raw bytes
with open("document.pdf", "rb") as f:
pdf_bytes = f.read()
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "file",
"file": pdf_bytes,
"mime_type": "application/pdf" # recommended for raw bytes (auto-detected from extension for file paths)
}
)
```
The `file` field accepts:
- **File path** (`str` or `pathlib.Path`) — LiteLLM reads the file and detects the MIME type from the extension
- **File object** (binary file-like object) — e.g. `open("doc.pdf", "rb")`
- **Raw bytes** (`bytes`) — use `mime_type` to specify the content type
LiteLLM automatically converts file inputs to base64 data URIs internally, so all providers work seamlessly.
### Using Base64 Encoded Documents
```python
@ -121,7 +167,7 @@ litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
Test request
**Test request — JSON body**
```bash
curl http://0.0.0.0:4000/v1/ocr \
@ -136,6 +182,27 @@ curl http://0.0.0.0:4000/v1/ocr \
}'
```
**Test request — multipart file upload**
Upload a file directly using multipart form data. No need to base64-encode the file yourself.
```bash
curl http://0.0.0.0:4000/v1/ocr \
-H "Authorization: Bearer sk-1234" \
-F "model=mistral-ocr" \
-F "file=@/path/to/document.pdf"
```
You can also pass optional parameters as additional form fields:
```bash
curl http://0.0.0.0:4000/v1/ocr \
-H "Authorization: Bearer sk-1234" \
-F "model=mistral-ocr" \
-F "file=@screenshot.png" \
-F 'pages=[0,1,2]' \
-F "include_image_base64=true"
```
## **Request/Response Format**
@ -168,10 +235,12 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) |
| `document` | object | Yes | Document to process. Must contain `type` and URL field |
| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images |
| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) |
| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) |
| `document` | object | Yes | Document to process. Must contain `type` and the corresponding field |
| `document.type` | string | Yes | `"document_url"` for PDFs/docs, `"image_url"` for images, or `"file"` for local files |
| `document.document_url` | string | Conditional | URL or data URI to the document (required if `type` is `"document_url"`) |
| `document.image_url` | string | Conditional | URL or data URI to the image (required if `type` is `"image_url"`) |
| `document.file` | string/bytes/file | Conditional | File path, bytes, or file-like object (required if `type` is `"file"`) |
| `document.mime_type` | string | No | Explicit MIME type for file inputs (auto-detected from extension if not provided) |
| `pages` | array | No | List of specific page indices to process (0-indexed) |
| `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings |
| `image_limit` | integer | No | Maximum number of images to return |
@ -179,7 +248,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
#### Document Format Examples
**For PDFs and documents:**
**For PDFs and documents (URL):**
```json
{
"type": "document_url",
@ -187,7 +256,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
}
```
**For images:**
**For images (URL):**
```json
{
"type": "image_url",
@ -203,6 +272,21 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
}
```
**For local files (SDK):**
```python
{"type": "file", "file": "/path/to/document.pdf"}
{"type": "file", "file": open("image.png", "rb")}
{"type": "file", "file": pdf_bytes, "mime_type": "application/pdf"}
```
**For file uploads (Proxy — multipart form):**
```bash
curl http://0.0.0.0:4000/v1/ocr \
-H "Authorization: Bearer sk-1234" \
-F "model=mistral-ocr" \
-F "file=@document.pdf"
```
### Response Format
The response follows Mistral's OCR format with the following structure:

179
docs/my-website/docs/pass_through/assembly_ai.md
View file

@ -1,31 +1,36 @@
# Assembly AI
# AssemblyAI
Pass-through endpoints for Assembly AI - call Assembly AI endpoints, in native format (no translation).
Pass-through endpoints for AssemblyAI - call AssemblyAI endpoints, in native format (no translation).
| Feature | Supported | Notes |
| Feature | Supported | Notes |
|-------|-------|-------|
| Cost Tracking | ✅ | works across all integrations |
| Logging | ✅ | works across all integrations |
Supports **ALL** Assembly AI Endpoints
Supports **ALL** AssemblyAI Endpoints
[**See All Assembly AI Endpoints**](https://www.assemblyai.com/docs/api-reference)
[**See All AssemblyAI Endpoints**](https://www.assemblyai.com/docs/api-reference)
<iframe width="840" height="500" src="https://www.loom.com/embed/aac3f4d74592448992254bfa79b9f62d?sid=267cd0ab-d92b-42fa-b97a-9f385ef8930c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## Supported Routes
| AssemblyAI Service | LiteLLM Route | AssemblyAI Base URL |
|-------------------|---------------|---------------------|
| Speech-to-Text (US) | `/assemblyai/*` | `api.assemblyai.com` |
| Speech-to-Text (EU) | `/eu.assemblyai/*` | `eu.api.assemblyai.com` |
## Quick Start
Let's call the Assembly AI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts)
Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts)
1. Add Assembly AI API Key to your environment
1. Add AssemblyAI API Key to your environment
```bash
export ASSEMBLYAI_API_KEY=""
```
2. Start LiteLLM Proxy
2. Start LiteLLM Proxy
```bash
litellm
@ -33,53 +38,157 @@ litellm
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
3. Test it!
Let's call the Assembly AI `/v2/transcripts` endpoint
Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts). Includes commented-out [Speech Understanding](https://www.assemblyai.com/docs/speech-understanding) features you can toggle on.
```python
import assemblyai as aai
LITELLM_VIRTUAL_KEY = "sk-1234" # <your-virtual-key>
LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}"
aai.settings.base_url = LITELLM_PROXY_BASE_URL
# Use a publicly-accessible URL
audio_file = "https://assembly.ai/wildfires.mp3"
# URL of the file to transcribe
FILE_URL = "https://assembly.ai/wildfires.mp3"
# Or use a local file:
# audio_file = "./example.mp3"
# You can also transcribe a local file by passing in a file path
# FILE_URL = './path/to/file.mp3'
config = aai.TranscriptionConfig(
speech_models=["universal-3-pro", "universal-2"],
language_detection=True,
speaker_labels=True,
# Speech understanding features
# sentiment_analysis=True,
# entity_detection=True,
# auto_chapters=True,
# summarization=True,
# summary_type=aai.SummarizationType.bullets,
# redact_pii=True,
# content_safety=True,
)
transcriber = aai.Transcriber()
transcript = transcriber.transcribe(FILE_URL)
print(transcript)
print(transcript.id)
transcript = aai.Transcriber().transcribe(audio_file, config=config)
if transcript.status == aai.TranscriptStatus.error:
raise RuntimeError(f"Transcription failed: {transcript.error}")
print(f"\nFull Transcript:\n\n{transcript.text}")
# Optionally print speaker diarization results
# for utterance in transcript.utterances:
# print(f"Speaker {utterance.speaker}: {utterance.text}")
```
## Calling Assembly AI EU endpoints
4. [Prompting with Universal-3 Pro](https://www.assemblyai.com/docs/speech-to-text/prompting) (optional)
If you want to send your request to the Assembly AI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `<your-proxy-base-url>/eu.assemblyai`
```python
import assemblyai as aai
aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
audio_file = "https://assemblyaiassets.com/audios/verbatim.mp3"
config = aai.TranscriptionConfig(
speech_models=["universal-3-pro", "universal-2"],
language_detection=True,
prompt="Produce a transcript suitable for conversational analysis. Every disfluency is meaningful data. Include: fillers (um, uh, er, ah, hmm, mhm, like, you know, I mean), repetitions (I I, the the), restarts (I was- I went), stutters (th-that, b-but, no-not), and informal speech (gonna, wanna, gotta)",
)
transcript = aai.Transcriber().transcribe(audio_file, config)
print(transcript.text)
```
## Calling AssemblyAI EU endpoints
If you want to send your request to the AssemblyAI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `<your-proxy-base-url>/eu.assemblyai`
```python
import assemblyai as aai
LITELLM_VIRTUAL_KEY = "sk-1234" # <your-virtual-key>
LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/eu.assemblyai" # <your-proxy-base-url>/eu.assemblyai
aai.settings.base_url = "http://0.0.0.0:4000/eu.assemblyai" # <your-proxy-base-url>/eu.assemblyai
aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}"
aai.settings.base_url = LITELLM_PROXY_BASE_URL
# Use a publicly-accessible URL
audio_file = "https://assembly.ai/wildfires.mp3"
# URL of the file to transcribe
FILE_URL = "https://assembly.ai/wildfires.mp3"
# You can also transcribe a local file by passing in a file path
# FILE_URL = './path/to/file.mp3'
# Or use a local file:
# audio_file = "./path/to/file.mp3"
transcriber = aai.Transcriber()
transcript = transcriber.transcribe(FILE_URL)
transcript = transcriber.transcribe(audio_file)
print(transcript)
print(transcript.id)
```
## LLM Gateway
Use AssemblyAI's [LLM Gateway](https://www.assemblyai.com/docs/llm-gateway) as an OpenAI-compatible provider — a unified API for Claude, GPT, and Gemini models with full LiteLLM logging, guardrails, and cost tracking support.
[**See Available Models**](https://www.assemblyai.com/docs/llm-gateway#available-models)
### Usage
#### LiteLLM Python SDK
```python
import litellm
import os
os.environ["ASSEMBLYAI_API_KEY"] = "your-assemblyai-api-key"
response = litellm.completion(
model="assemblyai/claude-sonnet-4-5-20250929",
messages=[{"role": "user", "content": "What is the capital of France?"}]
)
print(response.choices[0].message.content)
```
#### LiteLLM Proxy
1. Config
```yaml
model_list:
- model_name: assemblyai/*
litellm_params:
model: assemblyai/*
api_key: os.environ/ASSEMBLYAI_API_KEY
```
2. Start proxy
```bash
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
```python
import requests
headers = {
"authorization": "Bearer sk-1234" # Bearer <your-virtual-key>
}
response = requests.post(
"http://0.0.0.0:4000/v1/chat/completions",
headers=headers,
json={
"model": "assemblyai/claude-sonnet-4-5-20250929",
"messages": [
{"role": "user", "content": "What is the capital of France?"}
],
"max_tokens": 1000
}
)
result = response.json()
print(result["choices"][0]["message"]["content"])
```

157
docs/my-website/docs/pass_through/cursor.md Normal file
View file

@ -0,0 +1,157 @@
import Image from '@theme/IdealImage';
# Cursor Cloud Agents
Pass-through endpoints for the [Cursor Cloud Agents API](https://docs.cursor.com/account/api) — launch and manage cloud agents that work on your repositories, in native format (no translation).
| Feature | Supported | Notes |
|---------|-----------|-------|
| Cost Tracking | ✅ | Logged as $0.00 (subscription-based, no per-request pricing) |
| Logging | ✅ | All requests logged with operation classification |
| End-user Tracking | ❌ | [Tell us if you need this](https://github.com/BerriAI/litellm/issues/new) |
| Streaming | ❌ | Cursor API does not use streaming |
Just replace `https://api.cursor.com` with `LITELLM_PROXY_BASE_URL/cursor` 🚀
**Supported endpoints:**
| Endpoint | Method | Description |
|----------|--------|-------------|
| `/v0/agents` | GET | List agents |
| `/v0/agents` | POST | Launch an agent |
| `/v0/agents/{id}` | GET | Agent status |
| `/v0/agents/{id}` | DELETE | Delete an agent |
| `/v0/agents/{id}/conversation` | GET | Agent conversation |
| `/v0/agents/{id}/followup` | POST | Add follow-up |
| `/v0/agents/{id}/stop` | POST | Stop an agent |
| `/v0/me` | GET | API key info |
| `/v0/models` | GET | List models |
| `/v0/repositories` | GET | List GitHub repositories |
## Quick Start
### 1. Add Cursor API Key on the UI
Navigate to **Models + Endpoints → LLM Credentials** and click **Add Credential**. Select **Cursor** from the provider dropdown — you'll see the Cursor logo. Enter your API key from [cursor.com/settings](https://cursor.com/settings).
<Image img={require('../../img/cursor_add_credential.png')} alt="Add Cursor credential with logo" style={{maxWidth: '800px'}} />
### 2. Launch a Cursor Agent
```bash
curl -X POST http://0.0.0.0:4000/cursor/v0/agents \
-H "Authorization: Bearer <your-litellm-key>" \
-H "Content-Type: application/json" \
-d '{
"prompt": {
"text": "Add a README.md with installation instructions"
},
"source": {
"repository": "https://github.com/your-org/your-repo",
"ref": "main"
},
"target": {
"autoCreatePr": true
}
}'
```
**Expected Response:**
```json
{
"id": "bc_abc123",
"name": "Add README Documentation",
"status": "CREATING",
"source": {
"repository": "https://github.com/your-org/your-repo",
"ref": "main"
},
"target": {
"branchName": "cursor/add-readme-1234",
"url": "https://cursor.com/agents?id=bc_abc123",
"autoCreatePr": true
},
"createdAt": "2024-01-15T10:30:00Z"
}
```
### 3. View Logs
Navigate to **Logs** in the sidebar. Filter by "cursor" to see your agent requests. Each request shows the operation type (e.g., `cursor/cursor:agent:create`), status, duration, and cost.
<Image img={require('../../img/cursor_logs.png')} alt="Cursor requests in Logs page" style={{maxWidth: '800px'}} />
Click on any log entry to see full request details including provider, API base, and metadata.
<Image img={require('../../img/cursor_log_detail.png')} alt="Cursor log entry detail" style={{maxWidth: '800px'}} />
## Examples
Anything after `http://0.0.0.0:4000/cursor` is treated as a provider-specific route, and handled accordingly.
| **Original Endpoint** | **Replace With** |
|---|---|
| `https://api.cursor.com` | `http://0.0.0.0:4000/cursor` (LITELLM_PROXY_BASE_URL) |
| `-u YOUR_API_KEY:` (Basic Auth) | `-H "Authorization: Bearer <your-litellm-key>"` (LiteLLM Virtual Key) |
### List Available Models
```bash
curl http://0.0.0.0:4000/cursor/v0/models \
-H "Authorization: Bearer <your-litellm-key>"
```
### Check Agent Status
```bash
curl http://0.0.0.0:4000/cursor/v0/agents/bc_abc123 \
-H "Authorization: Bearer <your-litellm-key>"
```
### List All Agents
```bash
curl http://0.0.0.0:4000/cursor/v0/agents \
-H "Authorization: Bearer <your-litellm-key>"
```
### Add Follow-up to Agent
```bash
curl -X POST http://0.0.0.0:4000/cursor/v0/agents/bc_abc123/followup \
-H "Authorization: Bearer <your-litellm-key>" \
-H "Content-Type: application/json" \
-d '{
"prompt": {
"text": "Also add a section about troubleshooting"
}
}'
```
### Stop an Agent
```bash
curl -X POST http://0.0.0.0:4000/cursor/v0/agents/bc_abc123/stop \
-H "Authorization: Bearer <your-litellm-key>"
```
### Delete an Agent
```bash
curl -X DELETE http://0.0.0.0:4000/cursor/v0/agents/bc_abc123 \
-H "Authorization: Bearer <your-litellm-key>"
```
### Get API Key Info
```bash
curl http://0.0.0.0:4000/cursor/v0/me \
-H "Authorization: Bearer <your-litellm-key>"
```
## Related
- [Cursor Cloud Agents API Docs](https://docs.cursor.com/account/api)
- [Pass-through Endpoints Overview](./intro.md)
- [Virtual Keys](../proxy/virtual_keys.md)

71
docs/my-website/docs/pass_through/google_ai_studio.md
View file

@ -35,26 +35,25 @@ curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:countTokens?key=
```
</TabItem>
<TabItem value="js" label="Google AI Node.js SDK">
<TabItem value="js" label="Google GenAI JS SDK">
```javascript
const { GoogleGenerativeAI } = require("@google/generative-ai");
const { GoogleGenAI } = require("@google/genai");
const modelParams = {
model: 'gemini-pro',
};
const requestOptions = {
baseUrl: 'http://localhost:4000/gemini', // http://<proxy-base-url>/gemini
};
const genAI = new GoogleGenerativeAI("sk-1234"); // litellm proxy API key
const model = genAI.getGenerativeModel(modelParams, requestOptions);
const ai = new GoogleGenAI({
apiKey: "sk-1234", // litellm proxy API key
httpOptions: {
baseUrl: "http://localhost:4000/gemini", // http://<proxy-base-url>/gemini
},
});
async function main() {
try {
const result = await model.generateContent("Explain how AI works");
console.log(result.response.text());
const response = await ai.models.generateContent({
model: "gemini-2.5-flash",
contents: "Explain how AI works",
});
console.log(response.text);
} catch (error) {
console.error('Error:', error);
}
@ -63,12 +62,13 @@ async function main() {
// For streaming responses
async function main_streaming() {
try {
const streamingResult = await model.generateContentStream("Explain how AI works");
for await (const chunk of streamingResult.stream) {
console.log('Stream chunk:', JSON.stringify(chunk));
const response = await ai.models.generateContentStream({
model: "gemini-2.5-flash",
contents: "Explain how AI works",
});
for await (const chunk of response) {
process.stdout.write(chunk.text);
}
const aggregatedResponse = await streamingResult.response;
console.log('Aggregated response:', JSON.stringify(aggregatedResponse));
} catch (error) {
console.error('Error:', error);
}
@ -321,29 +321,28 @@ curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:generateContent?
```
</TabItem>
<TabItem value="js" label="Google AI Node.js SDK">
<TabItem value="js" label="Google GenAI JS SDK">
```javascript
const { GoogleGenerativeAI } = require("@google/generative-ai");
const { GoogleGenAI } = require("@google/genai");
const modelParams = {
model: 'gemini-pro',
};
const requestOptions = {
baseUrl: 'http://localhost:4000/gemini', // http://<proxy-base-url>/gemini
customHeaders: {
"tags": "gemini-js-sdk,pass-through-endpoint"
}
};
const genAI = new GoogleGenerativeAI("sk-1234");
const model = genAI.getGenerativeModel(modelParams, requestOptions);
const ai = new GoogleGenAI({
apiKey: "sk-1234",
httpOptions: {
baseUrl: "http://localhost:4000/gemini", // http://<proxy-base-url>/gemini
headers: {
"tags": "gemini-js-sdk,pass-through-endpoint",
},
},
});
async function main() {
try {
const result = await model.generateContent("Explain how AI works");
console.log(result.response.text());
const response = await ai.models.generateContent({
model: "gemini-2.5-flash",
contents: "Explain how AI works",
});
console.log(response.text);
} catch (error) {
console.error('Error:', error);
}

27
docs/my-website/docs/provider_registration/add_model_pricing.md
View file

@ -13,6 +13,7 @@ Here's the full specification with all available fields:
```json
{
"sample_spec": {
"aliases": ["optional list of alternate names for this model, e.g. dated versions like sample_spec-20250101"],
"code_interpreter_cost_per_session": 0.0,
"computer_use_input_cost_per_1k_tokens": 0.0,
"computer_use_output_cost_per_1k_tokens": 0.0,
@ -121,4 +122,28 @@ Here's the full specification with all available fields:
}
```
That's it! Your PR will be reviewed and merged.
### Using Aliases
Many providers release the same model under multiple names — for example, a `latest` tag and a dated version like `claude-sonnet-4-5-20250929`. Instead of duplicating the entire entry, you can use the `aliases` field:
```json
{
"claude-sonnet-4-5": {
"aliases": ["claude-sonnet-4-5-20250929"],
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true
}
}
```
At load time, each alias is expanded into a top-level entry sharing the same data as the canonical entry. The example above makes both `claude-sonnet-4-5` and `claude-sonnet-4-5-20250929` resolve with the same pricing and capabilities.
:::info
This is different from [`model_alias_map`](../completion/model_alias.md), which is a runtime SDK/proxy feature for mapping user-facing model names to LiteLLM model identifiers. The `aliases` field here is for the model cost JSON only — it avoids duplicate entries for models that share identical pricing and capabilities.
:::

7
docs/my-website/docs/providers/anthropic.md
View file

@ -4,6 +4,8 @@ import TabItem from '@theme/TabItem';
# Anthropic
LiteLLM supports all anthropic models.
- `claude-opus-4-6` (`claude-opus-4-6-20260205`)
- `claude-sonnet-4-6`
- `claude-sonnet-4-5-20250929`
- `claude-opus-4-5-20251101`
- `claude-opus-4-1-20250805`
@ -50,7 +52,7 @@ Check this in code, [here](../completion/input.md#translated-openai-params)
**Notes:**
- Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed.
- `response_format` is fully supported for Claude Sonnet 4.5 and Opus 4.1 models (see [Structured Outputs](#structured-outputs) section)
- `reasoning_effort` is automatically mapped to `output_config={"effort": ...}` for Claude Opus 4.5 models (see [Effort Parameter](./anthropic_effort.md))
- `reasoning_effort` is automatically mapped to `output_config={"effort": ...}` for Claude 4.6 and Opus 4.5 models (see [Effort Parameter](./anthropic_effort.md))
:::
@ -415,7 +417,10 @@ print(response)
| Model Name | Function Call |
|------------------|--------------------------------------------|
| claude-opus-4-6 | `completion('claude-opus-4-6-20260205', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-sonnet-4-5 | `completion('claude-sonnet-4-5-20250929', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-opus-4-5 | `completion('claude-opus-4-5-20251101', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-opus-4-1 | `completion('claude-opus-4-1-20250805', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-opus-4 | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-sonnet-4 | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3.7 | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']` |

118
docs/my-website/docs/providers/anthropic_effort.md
View file

@ -9,10 +9,11 @@ Control how many tokens Claude uses when responding with the `effort` parameter,
The `effort` parameter allows you to control how eager Claude is about spending tokens when responding to requests. This gives you the ability to trade off between response thoroughness and token efficiency, all with a single model.
**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. LiteLLM automatically adds the `effort-2025-11-24` beta header when:
- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only)
**Supported models:**
- **Claude 4.6** (Opus 4.6, Sonnet 4.6) — `output_config` is a stable API feature, no beta header needed. Opus 4.6 also supports `effort="max"`.
- **Claude Opus 4.5** — requires the `effort-2025-11-24` beta header (automatically added by LiteLLM).
For Claude Opus 4.5, `reasoning_effort="medium"`—both are automatically mapped to the correct format.
LiteLLM automatically maps `reasoning_effort` → `output_config={"effort": ...}` for all supported models.
## How Effort Works
@ -35,6 +36,7 @@ This gives a much greater degree of control over efficiency.
| Level | Description | Typical use case |
|-------|-------------|------------------|
| `max` | Maximum capability beyond high — Claude uses even more tokens for the most thorough outcome. **Only supported by Claude Opus 4.6.** | The hardest reasoning problems, complex multi-step research |
| `high` | Maximum capability—Claude uses as many tokens as needed for the best possible outcome. Equivalent to not setting the parameter. | Complex reasoning, difficult coding problems, agentic tasks |
| `medium` | Balanced approach with moderate token savings. | Agentic tasks that require a balance of speed, cost, and performance |
| `low` | Most efficient—significant token savings with some capability reduction. | Simpler tasks that need the best speed and lowest costs, such as subagents |
@ -49,16 +51,29 @@ This gives a much greater degree of control over efficiency.
```python
import litellm
# Works with Claude 4.6 models (no beta header needed)
response = litellm.completion(
model="anthropic/claude-sonnet-4-6",
messages=[{
"role": "user",
"content": "Analyze the trade-offs between microservices and monolithic architectures"
}],
reasoning_effort="medium" # Automatically mapped to output_config
)
print(response.choices[0].message.content)
```
```python
# Also works with Claude Opus 4.5 (beta header auto-injected)
response = litellm.completion(
model="anthropic/claude-opus-4-5-20251101",
messages=[{
"role": "user",
"content": "Analyze the trade-offs between microservices and monolithic architectures"
}],
reasoning_effort="medium" # Automatically mapped to output_config for Opus 4.5
reasoning_effort="medium"
)
print(response.choices[0].message.content)
```
</TabItem>
@ -71,8 +86,9 @@ const client = new Anthropic({
apiKey: process.env.ANTHROPIC_API_KEY,
});
// Claude 4.6 — output_config is a stable API feature (no beta header)
const response = await client.messages.create({
model: "claude-opus-4-5-20251101",
model: "claude-sonnet-4-6",
max_tokens: 4096,
messages: [{
role: "user",
@ -96,7 +112,29 @@ curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "anthropic/claude-opus-4-5-20251101",
"model": "anthropic/claude-sonnet-4-6",
"messages": [{
"role": "user",
"content": "Analyze the trade-offs between microservices and monolithic architectures"
}],
"reasoning_effort": "medium"
}'
```
### Direct Anthropic API Call
<Tabs>
<TabItem value="46" label="Claude 4.6 (stable)">
```bash
# Claude 4.6 — no beta header needed
curl https://api.anthropic.com/v1/messages \
--header "x-api-key: $ANTHROPIC_API_KEY" \
--header "anthropic-version: 2023-06-01" \
--header "content-type: application/json" \
--data '{
"model": "claude-sonnet-4-6",
"max_tokens": 4096,
"messages": [{
"role": "user",
"content": "Analyze the trade-offs between microservices and monolithic architectures"
@ -107,9 +145,11 @@ curl http://localhost:4000/v1/chat/completions \
}'
```
### Direct Anthropic API Call
</TabItem>
<TabItem value="45" label="Claude Opus 4.5 (beta)">
```bash
# Claude Opus 4.5 — requires beta header
curl https://api.anthropic.com/v1/messages \
--header "x-api-key: $ANTHROPIC_API_KEY" \
--header "anthropic-version: 2023-06-01" \
@ -128,10 +168,19 @@ curl https://api.anthropic.com/v1/messages \
}'
```
</TabItem>
</Tabs>
## Model Compatibility
The effort parameter is currently only supported by:
- **Claude Opus 4.5** (`claude-opus-4-5-20251101`)
The effort parameter is supported by:
- **Claude Opus 4.6** (`claude-opus-4-6`) — supports `high`, `medium`, `low`, and `max`
- **Claude Sonnet 4.6** (`claude-sonnet-4-6`) — supports `high`, `medium`, `low`
- **Claude Opus 4.5** (`claude-opus-4-5-20251101`) — supports `high`, `medium`, `low`
:::info
`effort="max"` is only available on Claude Opus 4.6. Using it with other models will raise a validation error.
:::
## When Should I Adjust the Effort Parameter?
@ -154,7 +203,7 @@ Example with tools:
import litellm
response = litellm.completion(
model="anthropic/claude-opus-4-5-20251101",
model="anthropic/claude-sonnet-4-6",
messages=[{
"role": "user",
"content": "Check the weather in multiple cities"
@ -173,9 +222,7 @@ response = litellm.completion(
}
}
}],
output_config={
"effort": "low" # Will make fewer tool calls
}
reasoning_effort="low" # Mapped to output_config — will make fewer tool calls
)
```
@ -187,18 +234,12 @@ The effort parameter works seamlessly with extended thinking. When both are enab
import litellm
response = litellm.completion(
model="anthropic/claude-opus-4-5-20251101",
model="anthropic/claude-sonnet-4-6",
messages=[{
"role": "user",
"content": "Solve this complex problem"
}],
thinking={
"type": "enabled",
"budget_tokens": 5000
},
output_config={
"effort": "medium" # Affects both thinking and response tokens
}
reasoning_effort="medium" # Mapped to adaptive thinking + output_config for 4.6 models
)
```
@ -218,14 +259,14 @@ response = litellm.completion(
The effort parameter is supported across all Anthropic-compatible providers:
- **Standard Anthropic API**: ✅ Supported (Claude Opus 4.5)
- **Azure Anthropic / Microsoft Foundry**: ✅ Supported (Claude Opus 4.5)
- **Amazon Bedrock**: ✅ Supported (Claude Opus 4.5)
- **Google Cloud Vertex AI**: ✅ Supported (Claude Opus 4.5)
- **Standard Anthropic API**: ✅ Supported (Claude 4.6, Opus 4.5)
- **Azure Anthropic / Microsoft Foundry**: ✅ Supported (Claude 4.6, Opus 4.5)
- **Amazon Bedrock**: ✅ Supported (Claude 4.6, Opus 4.5)
- **Google Cloud Vertex AI**: ✅ Supported (Claude 4.6, Opus 4.5)
LiteLLM automatically handles:
- Beta header injection (`effort-2025-11-24`) for all providers
- Parameter mapping: `reasoning_effort` → `output_config={"effort": ...}` for Claude Opus 4.5
- Parameter mapping: `reasoning_effort` → `output_config={"effort": ...}` for all supported models
- Beta header injection (`effort-2025-11-24`) only for Claude Opus 4.5 (not needed for 4.6 models)
## Usage and Pricing
@ -244,12 +285,13 @@ print(f"Total tokens: {response.usage.total_tokens}")
## Troubleshooting
### Beta header not being added
### Beta header not being added (Claude Opus 4.5)
LiteLLM automatically adds the `effort-2025-11-24` beta header when:
- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only)
LiteLLM automatically adds the `effort-2025-11-24` beta header for Claude Opus 4.5 when `reasoning_effort` or `output_config` is provided.
If you're not seeing the header:
**Note:** Claude 4.6 models do NOT need a beta header — `output_config` is a stable API feature for these models.
If you're not seeing the header for Opus 4.5:
1. Ensure you're using `reasoning_effort` parameter
2. Verify the model is Claude Opus 4.5
@ -257,7 +299,7 @@ If you're not seeing the header:
### Invalid effort value error
Only three values are accepted: `"high"`, `"medium"`, `"low"`. Any other value will raise a validation error:
Accepted values: `"high"`, `"medium"`, `"low"`, and `"max"` (Opus 4.6 only). Any other value will raise a validation error:
```python
# ❌ This will raise an error
@ -265,11 +307,17 @@ output_config={"effort": "very_low"}
# ✅ Use one of the valid values
output_config={"effort": "low"}
# ❌ This will raise an error (max only works on Opus 4.6)
litellm.completion(model="anthropic/claude-sonnet-4-6", reasoning_effort="max", ...)
# ✅ max is only for Opus 4.6
litellm.completion(model="anthropic/claude-opus-4-6", reasoning_effort="max", ...)
```
### Model not supported
Currently, only Claude Opus 4.5 supports the effort parameter. Using it with other models may result in the parameter being ignored or an error.
The effort parameter is supported by Claude Opus 4.6, Sonnet 4.6, and Opus 4.5. Using it with other models may result in the parameter being ignored or an error.
## Related Features

72
docs/my-website/docs/providers/azure_ai/azure_model_router.md
View file

@ -2,6 +2,32 @@
Azure Model Router is a feature in Azure AI Foundry that automatically routes your requests to the best available model based on your requirements. This allows you to use a single endpoint that intelligently selects the optimal model for each request.
## Quick Start
**Model pattern**: `azure_ai/model_router/<deployment-name>`
```python
import litellm
response = litellm.completion(
model="azure_ai/model_router/model-router", # Replace with your deployment name
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key="your-api-key",
)
```
**Proxy config** (`config.yaml`):
```yaml
model_list:
- model_name: model-router
litellm_params:
model: azure_ai/model_router/model-router
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/deployments/model-router/chat/completions?api-version=2025-01-01-preview
api_key: your-api-key
```
## Key Features
- **Automatic Model Selection**: Azure Model Router dynamically selects the best model for your request
@ -229,19 +255,51 @@ Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`), plus a fl
## Cost Tracking
LiteLLM automatically handles cost tracking for Azure Model Router by:
LiteLLM automatically handles cost tracking for Azure Model Router. Understanding how this works helps you interpret spend and debug billing.
1. **Detecting the actual model**: When Azure Model Router routes your request to a specific model (e.g., `gpt-4.1-nano-2025-04-14`), LiteLLM extracts this from the response
2. **Calculating accurate costs**: Costs are calculated based on:
- The actual model used (e.g., `gpt-4.1-nano` token costs)
- Plus a flat infrastructure cost of **$0.14 per million input tokens** for using the Model Router
3. **Streaming support**: Cost tracking works correctly for both streaming and non-streaming requests
### How LiteLLM Calculates Cost
When you use Azure Model Router, LiteLLM computes **two cost components**:
| Component | Description | When Applied |
|-----------|-------------|--------------|
| **Model Cost** | Token-based cost for the actual model that handled the request (e.g., `gpt-5-nano`, `gpt-4.1-nano`) | Always, when Azure returns the model in the response |
| **Router Flat Cost** | $0.14 per million input tokens (Azure AI Foundry infrastructure fee) | When the **request** was made via a model router endpoint |
### Cost Calculation Flow
1. **Request model detection**: LiteLLM records the model you requested (e.g., `azure_ai/model_router/model-router`). If it contains `model_router` or `model-router`, the request is treated as a router request.
2. **Response model extraction**: Azure returns the actual model used in the response (e.g., `gpt-5-nano-2025-08-07`). LiteLLM uses this for the model cost lookup.
3. **Model cost**: LiteLLM looks up the response model in its pricing table and computes cost from prompt tokens and completion tokens.
4. **Router flat cost**: Because the original request was to a model router, LiteLLM adds the flat cost ($0.14 per M input tokens) on top of the model cost.
5. **Total cost**: `Total = Model Cost + Router Flat Cost`
### Configuration Requirements
For cost tracking to work correctly:
- **Use the full pattern**: `azure_ai/model_router/<deployment-name>` (e.g., `azure_ai/model_router/model-router`)
- **Proxy config**: When using the LiteLLM proxy, set `model` in `litellm_params` to the full pattern so the request model is correctly identified as a router
```yaml
# proxy_server_config.yaml
model_list:
- model_name: model-router
litellm_params:
model: azure_ai/model_router/model-router # Required for router cost detection
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/deployments/model-router/chat/completions?api-version=2025-01-01-preview
api_key: your-api-key
```
### Cost Breakdown
When you use Azure Model Router, the total cost includes:
- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-4.1-nano`)
- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-5-nano`, `gpt-4.1-nano`)
- **Router Flat Cost**: $0.14 per million input tokens (Azure AI Foundry infrastructure fee)
### Example Response with Cost

4
docs/my-website/docs/providers/bedrock.md
View file

@ -660,7 +660,7 @@ Same as [Anthropic API response](../providers/anthropic#usage---thinking--reason
LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic-beta` header. This enables access to experimental features like:
- **1M Context Window** - Up to 1 million tokens of context (Claude Sonnet 4)
- **1M Context Window** - Up to 1 million tokens of context (Claude Opus 4.6, Sonnet 4.5, Sonnet 4)
- **Computer Use Tools** - AI that can interact with computer interfaces
- **Token-Efficient Tools** - More efficient tool usage patterns
- **Extended Output** - Up to 128K output tokens
@ -670,7 +670,7 @@ LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic
| Beta Feature | Header Value | Compatible Models | Description |
|--------------|-------------|------------------|-------------|
| 1M Context Window | `context-1m-2025-08-07` | Claude Sonnet 4 | Enable 1 million token context window |
| 1M Context Window | `context-1m-2025-08-07` | Claude Opus 4.6, Sonnet 4.5, Sonnet 4 | Enable 1 million token context window |
| Computer Use (Latest) | `computer-use-2025-01-24` | Claude 3.7 Sonnet | Latest computer use tools |
| Computer Use (Legacy) | `computer-use-2024-10-22` | Claude 3.5 Sonnet v2 | Computer use tools for Claude 3.5 |
| Token-Efficient Tools | `token-efficient-tools-2025-02-19` | Claude 3.7 Sonnet | More efficient tool usage |

2
docs/my-website/docs/providers/bedrock_agentcore.md
View file

@ -13,7 +13,7 @@ Call Bedrock AgentCore in the OpenAI Request/Response format.
:::info
This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers, add them as you would any other MCP server. See the [MCP documentation](https://docs.litellm.ai/docs/mcp) for details.
This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers with LiteLLM, see the [MCP AWS SigV4 Auth](https://docs.litellm.ai/docs/mcp_aws_sigv4) guide for setup instructions.
:::

157
docs/my-website/docs/providers/bedrock_mantle.md Normal file
View file

@ -0,0 +1,157 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Amazon Bedrock Mantle
[Amazon Bedrock Mantle](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html) is Amazon Bedrock's distributed inference engine (Project Mantle) that exposes an **OpenAI-compatible API** for Bedrock-hosted models.
Use this provider to call Bedrock Mantle models with accurate **AWS Bedrock pricing** instead of OpenAI pricing.
:::tip
**We support ALL Bedrock Mantle models, just set `model=bedrock_mantle/<model-id>` as a prefix when sending litellm requests**
:::
## API Key
```python
# env variable
os.environ['BEDROCK_MANTLE_API_KEY'] = "your-aws-bedrock-api-key"
# optional: override region (defaults to us-east-1)
os.environ['BEDROCK_MANTLE_REGION'] = "us-east-1" # or use AWS_REGION
```
## Supported Models
| Model | Context Window | Input (per 1M tokens) | Output (per 1M tokens) |
|-------|---------------|----------------------|------------------------|
| `openai.gpt-oss-120b` | 131K | $0.15 | $0.60 |
| `openai.gpt-oss-20b` | 131K | $0.075 | $0.30 |
| `openai.gpt-oss-safeguard-120b` | 131K | $0.15 | $0.60 |
| `openai.gpt-oss-safeguard-20b` | 131K | $0.075 | $0.30 |
## Sample Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key"
response = completion(
model="bedrock_mantle/openai.gpt-oss-120b",
messages=[{"role": "user", "content": "hello from litellm"}],
)
print(response)
```
</TabItem>
<TabItem value="streaming" label="Streaming">
```python
from litellm import completion
import os
os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key"
response = completion(
model="bedrock_mantle/openai.gpt-oss-120b",
messages=[{"role": "user", "content": "hello from litellm"}],
stream=True,
)
for chunk in response:
print(chunk)
```
</TabItem>
<TabItem value="async" label="Async">
```python
import asyncio
from litellm import acompletion
import os
os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key"
async def main():
response = await acompletion(
model="bedrock_mantle/openai.gpt-oss-120b",
messages=[{"role": "user", "content": "hello from litellm"}],
)
print(response)
asyncio.run(main())
```
</TabItem>
</Tabs>
## Region Configuration
The API base URL is `https://bedrock-mantle.{region}.api.aws/v1`. Region is resolved in this order:
1. `BEDROCK_MANTLE_REGION` env var
2. `AWS_REGION` env var
3. Default: `us-east-1`
**Supported regions:** `us-east-1`, `us-east-2`, `us-west-2`, `eu-west-1`, `eu-west-2`, `eu-central-1`, `eu-south-1`, `eu-north-1`, `ap-northeast-1`, `ap-south-1`, `ap-southeast-3`, `sa-east-1`
```python
import os
os.environ['BEDROCK_MANTLE_REGION'] = "eu-west-1"
# or pass api_base directly
response = completion(
model="bedrock_mantle/openai.gpt-oss-120b",
messages=[{"role": "user", "content": "hello"}],
api_base="https://bedrock-mantle.eu-west-1.api.aws/v1",
)
```
## Usage with LiteLLM Proxy
### 1. Set Bedrock Mantle models on config.yaml
```yaml
model_list:
- model_name: gpt-oss-120b
litellm_params:
model: bedrock_mantle/openai.gpt-oss-120b
api_key: os.environ/BEDROCK_MANTLE_API_KEY
# optional region override:
api_base: "https://bedrock-mantle.us-east-1.api.aws/v1"
- model_name: gpt-oss-20b
litellm_params:
model: bedrock_mantle/openai.gpt-oss-20b
api_key: os.environ/BEDROCK_MANTLE_API_KEY
```
### 2. Start the proxy
```shell
litellm --config /path/to/config.yaml
```
### 3. Send a request
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000",
)
response = client.chat.completions.create(
model="gpt-oss-120b",
messages=[{"role": "user", "content": "hello from litellm"}],
)
print(response)
```

291
docs/my-website/docs/providers/black_forest_labs.md Normal file
View file

@ -0,0 +1,291 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Black Forest Labs Image Generation
Black Forest Labs provides state-of-the-art text-to-image generation using their FLUX models.
## Overview
| Property | Details |
|----------|---------|
| Description | Black Forest Labs FLUX models for high-quality text-to-image generation |
| Provider Route on LiteLLM | `black_forest_labs/` |
| Provider Doc | [Black Forest Labs API ↗](https://docs.bfl.ai/) |
| Supported Operations | [`/images/generations`](#image-generation) |
## Setup
### API Key
```python showLineNumbers
import os
# Set your Black Forest Labs API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
```
Get your API key from [Black Forest Labs](https://blackforestlabs.ai/).
## Supported Models
| Model Name | Description | Price |
|------------|-------------|-------|
| `black_forest_labs/flux-pro-1.1` | Fast & reliable standard generation | $0.04/image |
| `black_forest_labs/flux-pro-1.1-ultra` | Ultra high-resolution (up to 4MP) | $0.06/image |
| `black_forest_labs/flux-dev` | Development/open-source variant | $0.025/image |
| `black_forest_labs/flux-pro` | Original pro model | $0.05/image |
## Image Generation
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="basic" label="Basic Usage">
```python showLineNumbers title="Basic Image Generation"
import os
import litellm
# Set your API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
# Generate an image
response = litellm.image_generation(
model="black_forest_labs/flux-pro-1.1",
prompt="A beautiful sunset over the ocean with sailing boats",
)
# BFL returns URLs
print(response.data[0].url)
```
</TabItem>
<TabItem value="async" label="Async Usage">
```python showLineNumbers title="Async Image Generation"
import os
import asyncio
import litellm
# Set your API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
async def generate_image():
response = await litellm.aimage_generation(
model="black_forest_labs/flux-pro-1.1",
prompt="A futuristic city skyline at night",
)
print(response.data[0].url)
# Run the async function
asyncio.run(generate_image())
```
</TabItem>
<TabItem value="size" label="Custom Size">
```python showLineNumbers title="Image Generation with Custom Size"
import os
import litellm
# Set your API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
# Generate with specific dimensions
response = litellm.image_generation(
model="black_forest_labs/flux-pro-1.1",
prompt="A majestic mountain landscape",
size="1792x1024", # Maps to width/height
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="ultra" label="Ultra High-Res">
```python showLineNumbers title="Ultra High Resolution with flux-pro-1.1-ultra"
import os
import litellm
# Set your API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
# Generate ultra high-resolution image
response = litellm.image_generation(
model="black_forest_labs/flux-pro-1.1-ultra",
prompt="Detailed portrait of a fantasy character",
size="2048x2048", # Up to 4MP supported
quality="hd", # Maps to raw=True for natural look
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="advanced" label="Advanced Parameters">
```python showLineNumbers title="Advanced Image Generation with BFL Parameters"
import os
import litellm
# Set your API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
# Generate with BFL-specific parameters
response = litellm.image_generation(
model="black_forest_labs/flux-pro-1.1",
prompt="A cute orange cat sitting on a windowsill",
seed=42, # For reproducible results
output_format="png", # png or jpeg
safety_tolerance=2, # 0-6, higher = more permissive
prompt_upsampling=True, # Enhance prompt for better results
)
print(response.data[0].url)
```
</TabItem>
</Tabs>
### Usage - LiteLLM Proxy Server
#### 1. Configure your config.yaml
```yaml showLineNumbers title="Black Forest Labs Image Generation Configuration"
model_list:
- model_name: flux-pro
litellm_params:
model: black_forest_labs/flux-pro-1.1
api_key: os.environ/BFL_API_KEY
model_info:
mode: image_generation
- model_name: flux-ultra
litellm_params:
model: black_forest_labs/flux-pro-1.1-ultra
api_key: os.environ/BFL_API_KEY
model_info:
mode: image_generation
- model_name: flux-dev
litellm_params:
model: black_forest_labs/flux-dev
api_key: os.environ/BFL_API_KEY
model_info:
mode: image_generation
general_settings:
master_key: sk-1234
```
#### 2. Start LiteLLM Proxy Server
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### 3. Make image generation requests
<Tabs>
<TabItem value="openai-sdk" label="OpenAI SDK">
```python showLineNumbers title="Black Forest Labs via Proxy - OpenAI SDK"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000",
api_key="sk-1234"
)
# Generate image with FLUX Pro
response = client.images.generate(
model="flux-pro",
prompt="A beautiful garden with colorful flowers",
size="1024x1024",
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Black Forest Labs via Proxy - cURL"
curl -X POST 'http://localhost:4000/v1/images/generations' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "flux-pro",
"prompt": "A beautiful garden with colorful flowers",
"size": "1024x1024"
}'
```
</TabItem>
</Tabs>
## Supported Parameters
### OpenAI-Compatible Parameters
| Parameter | Type | Description | Mapping |
|-----------|------|-------------|---------|
| `prompt` | string | Text description of the image to generate | Direct |
| `model` | string | The FLUX model to use | Direct |
| `size` | string | Image dimensions (e.g., `1024x1024`) | Maps to `width` and `height` |
| `n` | integer | Number of images (ultra model only, up to 4) | Maps to `num_images` |
| `quality` | string | `hd` for natural look | Maps to `raw=True` for ultra |
| `response_format` | string | `url` or `b64_json` | Direct |
### Black Forest Labs Specific Parameters
| Parameter | Type | Description | Default |
|-----------|------|-------------|---------|
| `width` | integer | Image width (256-1920, multiples of 16) | 1024 |
| `height` | integer | Image height (256-1920, multiples of 16) | 1024 |
| `aspect_ratio` | string | Alternative to width/height (e.g., `16:9`, `1:1`) | - |
| `seed` | integer | Seed for reproducible results | Random |
| `output_format` | string | Output format: `png` or `jpeg` | `png` |
| `safety_tolerance` | integer | Safety filter tolerance (0-6, higher = more permissive) | 2 |
| `prompt_upsampling` | boolean | Enhance prompt for better results | `false` |
### Ultra Model Specific Parameters
| Parameter | Type | Description | Default |
|-----------|------|-------------|---------|
| `raw` | boolean | Raw mode for more natural, less synthetic look | `false` |
| `num_images` | integer | Number of images to generate (1-4) | 1 |
## How It Works
Black Forest Labs uses a polling-based API:
1. **Submit Request**: LiteLLM sends your prompt to BFL
2. **Get Task ID**: BFL returns a task ID and polling URL
3. **Poll for Result**: LiteLLM automatically polls until the image is ready
4. **Return Result**: The generated image URL is returned
This polling is handled automatically by LiteLLM - you just call `image_generation()` and get the result.
## Getting Started
1. Create an account at [Black Forest Labs](https://blackforestlabs.ai/)
2. Get your API key from the dashboard
3. Set your `BFL_API_KEY` environment variable
4. Use `litellm.image_generation()` with any supported model
## Additional Resources
- [Black Forest Labs Documentation](https://docs.bfl.ai/)
- [Black Forest Labs Image Editing](./black_forest_labs_img_edit.md) - For editing existing images
- [FLUX Model Information](https://blackforestlabs.ai/)

301
docs/my-website/docs/providers/black_forest_labs_img_edit.md Normal file
View file

@ -0,0 +1,301 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Black Forest Labs Image Editing
Black Forest Labs provides powerful image editing capabilities using their FLUX models to modify existing images based on text descriptions.
## Overview
| Property | Details |
|----------|---------|
| Description | Black Forest Labs Image Editing uses FLUX Kontext and other models to modify, inpaint, and expand images based on text prompts. |
| Provider Route on LiteLLM | `black_forest_labs/` |
| Provider Doc | [Black Forest Labs API ↗](https://docs.bfl.ai/) |
| Supported Operations | [`/images/edits`](#image-editing) |
## Setup
### API Key
```python showLineNumbers
import os
# Set your Black Forest Labs API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
```
Get your API key from [Black Forest Labs](https://blackforestlabs.ai/).
## Supported Models
| Model Name | Description | Use Case |
|------------|-------------|----------|
| `black_forest_labs/flux-kontext-pro` | FLUX Kontext Pro - General image editing with prompts | General editing, style transfer |
| `black_forest_labs/flux-kontext-max` | FLUX Kontext Max - Premium quality editing | High-quality edits |
| `black_forest_labs/flux-pro-1.0-fill` | FLUX Pro Fill - Inpainting with mask | Remove/replace objects |
| `black_forest_labs/flux-pro-1.0-expand` | FLUX Pro Expand - Outpainting | Expand image borders |
## Image Editing
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="basic-edit" label="Basic Usage">
```python showLineNumbers title="Basic Image Editing"
import os
import litellm
# Set your API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
# Edit an image with a prompt
response = litellm.image_edit(
model="black_forest_labs/flux-kontext-pro",
image=open("path/to/your/image.png", "rb"),
prompt="Add a green leaf to the scene",
)
# BFL returns URLs
print(response.data[0].url)
```
</TabItem>
<TabItem value="async-edit" label="Async Usage">
```python showLineNumbers title="Async Image Editing"
import os
import asyncio
import litellm
# Set your API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
async def edit_image():
response = await litellm.aimage_edit(
model="black_forest_labs/flux-kontext-pro",
image=open("path/to/your/image.png", "rb"),
prompt="Make this image look like a watercolor painting",
)
print(response.data[0].url)
# Run the async function
asyncio.run(edit_image())
```
</TabItem>
<TabItem value="inpainting" label="Inpainting (Fill)">
```python showLineNumbers title="Inpainting with Mask"
import os
import litellm
# Set your API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
# Use flux-pro-1.0-fill for inpainting
response = litellm.image_edit(
model="black_forest_labs/flux-pro-1.0-fill",
image=open("path/to/your/image.png", "rb"),
mask=open("path/to/mask.png", "rb"), # White areas will be edited
prompt="Replace with a beautiful garden",
steps=50, # BFL-specific parameter
guidance=30, # BFL-specific parameter
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="outpainting" label="Outpainting (Expand)">
```python showLineNumbers title="Outpainting - Expand Image Borders"
import os
import litellm
# Set your API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
# Use flux-pro-1.0-expand to extend image borders
response = litellm.image_edit(
model="black_forest_labs/flux-pro-1.0-expand",
image=open("path/to/your/image.png", "rb"),
prompt="Continue the scene with a mountain landscape",
top=256, # Expand 256 pixels at top
bottom=256, # Expand 256 pixels at bottom
left=128, # Expand 128 pixels at left
right=128, # Expand 128 pixels at right
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="advanced" label="Advanced Parameters">
```python showLineNumbers title="Advanced Image Editing with BFL Parameters"
import os
import litellm
# Set your API key
os.environ["BFL_API_KEY"] = "your-api-key-here"
# Edit image with BFL-specific parameters
response = litellm.image_edit(
model="black_forest_labs/flux-kontext-pro",
image=open("path/to/your/image.png", "rb"),
prompt="Transform into cyberpunk style with neon lights",
seed=42, # For reproducible results
output_format="png", # png or jpeg
safety_tolerance=2, # 0-6, higher = more permissive
aspect_ratio="16:9", # Output aspect ratio
)
print(response.data[0].url)
```
</TabItem>
</Tabs>
### Usage - LiteLLM Proxy Server
#### 1. Configure your config.yaml
```yaml showLineNumbers title="Black Forest Labs Image Editing Configuration"
model_list:
- model_name: bfl-kontext-pro
litellm_params:
model: black_forest_labs/flux-kontext-pro
api_key: os.environ/BFL_API_KEY
model_info:
mode: image_edit
- model_name: bfl-kontext-max
litellm_params:
model: black_forest_labs/flux-kontext-max
api_key: os.environ/BFL_API_KEY
model_info:
mode: image_edit
- model_name: bfl-fill
litellm_params:
model: black_forest_labs/flux-pro-1.0-fill
api_key: os.environ/BFL_API_KEY
model_info:
mode: image_edit
- model_name: bfl-expand
litellm_params:
model: black_forest_labs/flux-pro-1.0-expand
api_key: os.environ/BFL_API_KEY
model_info:
mode: image_edit
general_settings:
master_key: sk-1234
```
#### 2. Start LiteLLM Proxy Server
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### 3. Make image editing requests
<Tabs>
<TabItem value="openai-sdk" label="OpenAI SDK">
```python showLineNumbers title="Black Forest Labs via Proxy - OpenAI SDK"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000",
api_key="sk-1234"
)
# Edit image with FLUX Kontext Pro
response = client.images.edit(
model="bfl-kontext-pro",
image=open("path/to/your/image.png", "rb"),
prompt="Add magical sparkles and fairy dust",
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Black Forest Labs via Proxy - cURL"
curl --location 'http://localhost:4000/v1/images/edits' \
--header 'Authorization: Bearer sk-1234' \
--form 'model="bfl-kontext-pro"' \
--form 'prompt="Add a sunset in the background"' \
--form 'image=@"path/to/your/image.png"'
```
</TabItem>
</Tabs>
## Supported Parameters
### OpenAI-Compatible Parameters
| Parameter | Type | Description | Default |
|-----------|------|-------------|---------|
| `image` | file | The image file to edit | Required |
| `prompt` | string | Text description of the desired changes | Required |
| `model` | string | The FLUX model to use | Required |
| `mask` | file | Mask image for inpainting (flux-pro-1.0-fill) | Optional |
| `n` | integer | Number of images (BFL returns 1 per request) | `1` |
| `size` | string | Maps to aspect_ratio | Optional |
| `response_format` | string | `url` or `b64_json` | `url` |
### Black Forest Labs Specific Parameters
| Parameter | Type | Description | Default | Models |
|-----------|------|-------------|---------|--------|
| `seed` | integer | Seed for reproducible results | Random | All |
| `output_format` | string | Output format: `png` or `jpeg` | `png` | All |
| `safety_tolerance` | integer | Safety filter tolerance (0-6) | 2 | All |
| `aspect_ratio` | string | Output aspect ratio (e.g., `16:9`, `1:1`) | Original | Kontext models |
| `steps` | integer | Number of inference steps | Model default | Fill |
| `guidance` | float | Guidance scale | Model default | Fill |
| `grow_mask` | integer | Pixels to grow mask | 0 | Fill |
| `top` | integer | Pixels to expand at top | 0 | Expand |
| `bottom` | integer | Pixels to expand at bottom | 0 | Expand |
| `left` | integer | Pixels to expand at left | 0 | Expand |
| `right` | integer | Pixels to expand at right | 0 | Expand |
## How It Works
Black Forest Labs uses a polling-based API:
1. **Submit Request**: LiteLLM sends your image and prompt to BFL
2. **Get Task ID**: BFL returns a task ID and polling URL
3. **Poll for Result**: LiteLLM automatically polls until the image is ready
4. **Return Result**: The generated image URL is returned
This polling is handled automatically by LiteLLM - you just call `image_edit()` and get the result.
## Getting Started
1. Create an account at [Black Forest Labs](https://blackforestlabs.ai/)
2. Get your API key from the dashboard
3. Set your `BFL_API_KEY` environment variable
4. Use `litellm.image_edit()` with any supported model
## Additional Resources
- [Black Forest Labs Documentation](https://docs.bfl.ai/)
- [FLUX Model Information](https://blackforestlabs.ai/)

36
docs/my-website/docs/providers/chatgpt.md
View file

@ -4,12 +4,12 @@ Use ChatGPT Pro/Max subscription models through LiteLLM with OAuth device flow a
| Property | Details |
|-------|-------|
| Description | ChatGPT subscription access (Codex + GPT-5.2 family) via ChatGPT backend API |
| Description | ChatGPT subscription access (Codex + GPT-5.3/5.4 family) via ChatGPT backend API |
| Provider Route on LiteLLM | `chatgpt/` |
| Supported Endpoints | `/responses`, `/chat/completions` (bridged to Responses for supported models) |
| API Reference | https://chatgpt.com |
ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.2`).
ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.4`).
Notes:
- The ChatGPT subscription backend rejects token limit fields (`max_tokens`, `max_output_tokens`, `max_completion_tokens`) and `metadata`. LiteLLM strips these fields for this provider.
@ -31,7 +31,7 @@ ChatGPT subscription access uses an OAuth device code flow:
import litellm
response = litellm.responses(
model="chatgpt/gpt-5.2-codex",
model="chatgpt/gpt-5.3-codex",
input="Write a Python hello world"
)
@ -44,7 +44,7 @@ print(response)
import litellm
response = litellm.completion(
model="chatgpt/gpt-5.2",
model="chatgpt/gpt-5.4",
messages=[{"role": "user", "content": "Write a Python hello world"}]
)
@ -55,16 +55,36 @@ print(response)
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: chatgpt/gpt-5.2
- model_name: chatgpt/gpt-5.4
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.2
- model_name: chatgpt/gpt-5.2-codex
model: chatgpt/gpt-5.4
- model_name: chatgpt/gpt-5.4-pro
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.2-codex
model: chatgpt/gpt-5.4-pro
- model_name: chatgpt/gpt-5.3-codex
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-codex
- model_name: chatgpt/gpt-5.3-codex-spark
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-codex-spark
- model_name: chatgpt/gpt-5.3-instant
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-instant
- model_name: chatgpt/gpt-5.3-chat-latest
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-chat-latest
```
```bash showLineNumbers title="Start LiteLLM Proxy"

23
docs/my-website/docs/providers/gemini.md
View file

@ -1562,13 +1562,18 @@ LiteLLM Supports the following image types passed in `url`
## Media Resolution Control (Images & Videos)
For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types.
LiteLLM supports OpenAI's `detail` parameter for specifying the image resolution when using Gemini models. The behavior differs between Gemini versions:
| Gemini Version | Resolution Control | Behavior |
|----------------|-------------------|----------|
| Gemini 3+ | Per-part | Each image/video can have its own `detail` setting |
| Gemini 2.x (2.0, 2.5) | Global | The highest `detail` from all images is applied globally via `mediaResolution` in `generationConfig` |
**Supported `detail` values:**
- `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos)
- `"medium"` - Maps to `media_resolution: "medium"`
- `"high"` - Maps to `media_resolution: "high"` (1120 tokens for images)
- `"ultra_high"` - Maps to `media_resolution: "ultra_high"`
- `"low"` - Maps to `MEDIA_RESOLUTION_LOW` (280 tokens for images, 70 tokens per frame for videos)
- `"medium"` - Maps to `MEDIA_RESOLUTION_MEDIUM`
- `"high"` - Maps to `MEDIA_RESOLUTION_HIGH` (1120 tokens for images)
- `"ultra_high"` - Maps to `MEDIA_RESOLUTION_ULTRA_HIGH`
- `"auto"` or `None` - Model decides optimal resolution (no `media_resolution` set)
**Usage Examples:**
@ -1605,8 +1610,9 @@ messages = [
}
]
# Works with both Gemini 2.x and 3+
response = completion(
model="gemini/gemini-3-pro-preview",
model="gemini/gemini-2.5-flash", # or gemini-3-pro-preview
messages=messages,
)
```
@ -1647,7 +1653,9 @@ response = completion(
</Tabs>
:::info
**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types, and is only available for Gemini 3+ models.
**Gemini 3+ Per-Part Resolution:** Each image or video can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This works with both `image_url` and `file` content types.
**Gemini 2.x Global Resolution:** When multiple images have different `detail` values, LiteLLM uses the highest resolution found and applies it globally via `mediaResolution` in `generationConfig` (e.g., if one image has `"low"` and another has `"high"`, all images will use `"high"`).
:::
## Video Metadata Control
@ -2041,6 +2049,7 @@ response = litellm.completion(
| gemini-2.0-flash-lite-preview-02-05 | `completion(model='gemini/gemini-2.0-flash-lite-preview-02-05', messages)` | `os.environ['GEMINI_API_KEY']` |
| gemini-2.5-flash-preview-09-2025 | `completion(model='gemini/gemini-2.5-flash-preview-09-2025', messages)` | `os.environ['GEMINI_API_KEY']` |
| gemini-2.5-flash-lite-preview-09-2025 | `completion(model='gemini/gemini-2.5-flash-lite-preview-09-2025', messages)` | `os.environ['GEMINI_API_KEY']` |
| gemini-3.1-flash-lite-preview | `completion(model='gemini/gemini-3.1-flash-lite-preview', messages)` | `os.environ['GEMINI_API_KEY']` |
| gemini-flash-latest | `completion(model='gemini/gemini-flash-latest', messages)` | `os.environ['GEMINI_API_KEY']` |
| gemini-flash-lite-latest | `completion(model='gemini/gemini-flash-lite-latest', messages)` | `os.environ['GEMINI_API_KEY']` |

1
docs/my-website/docs/providers/groq.md
View file

@ -159,6 +159,7 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion
| moonshotai/kimi-k2-instruct-0905 | `completion(model="groq/moonshotai/kimi-k2-instruct-0905", messages)` |
| openai/gpt-oss-120b | `completion(model="groq/openai/gpt-oss-120b", messages)` |
| openai/gpt-oss-20b | `completion(model="groq/openai/gpt-oss-20b", messages)` |
| openai/gpt-oss-safeguard-20b | `completion(model="groq/openai/gpt-oss-safeguard-20b", messages)` |
## Groq - Tool / Function Calling Example

Some files were not shown because too many files have changed in this diff Show more