scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge litellm_ui_demo_usage_page into litellm_dev_02_05_2025_p1

Browse source
This commit is contained in:
Ishaan Jaffer 2026-02-05 16:09:09 -08:00
commit a65476edaf
5773 changed files with 1124621 additions and 106826 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2858
.circleci/config.yml
View file

File diff suppressed because it is too large Load diff

17
.circleci/requirements.txt
View file

@ -1,13 +1,20 @@
# used by CI/CD testing
openai==1.54.0
openai==1.100.1
python-dotenv
tiktoken
importlib_metadata
cohere
redis
redis==5.2.1
redisvl==0.4.1
anthropic
orjson==3.9.15
pydantic==2.7.1
orjson==3.10.12 # fast /embedding responses
pydantic==2.11.0
google-cloud-aiplatform==1.43.0
fastapi-sso==0.10.0
google-cloud-iam==2.19.1
fastapi-sso==0.16.0
uvloop==0.21.0
mcp==1.25.0 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
fastuuid==0.12.0
responses==0.25.7 # for proxy client tests
pytest-retry==1.6.3 # for automatic test retries

11
.devcontainer/devcontainer.json
View file

@ -11,7 +11,12 @@
// },
// Features to add to the dev container. More info: https://containers.dev/features.
// "features": {},
"features": {
"ghcr.io/devcontainers/features/node:1": {
"version": "lts"
},
"ghcr.io/devcontainers/features/docker-in-docker:2": {}
},
// Configure tool-specific properties.
"customizations": {
@ -30,7 +35,7 @@
// Use 'forwardPorts' to make a list of ports inside the container available locally.
"forwardPorts": [4000],
"containerEnv": {
"LITELLM_LOG": "DEBUG"
},
@ -48,5 +53,5 @@
// "remoteUser": "litellm",
// Use 'postCreateCommand' to run commands after the container is created.
"postCreateCommand": "pipx install poetry && poetry install -E extra_proxy -E proxy"
"postCreateCommand": "bash ./.devcontainer/post-create.sh"
}

17
.devcontainer/post-create.sh Normal file
View file

@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -e
echo "[post-create] Installing poetry via pip"
python -m pip install --upgrade pip
python -m pip install poetry
echo "[post-create] Installing Python dependencies (poetry)"
poetry install --with dev --extras proxy
echo "[post-create] Generating Prisma client"
poetry run prisma generate
echo "[post-create] Installing npm dependencies"
cd ui/litellm-dashboard && npm install --no-audit --no-fund
echo "[post-create] Done"

46
.dockerignore
View file

@ -4,9 +4,51 @@ cookbook
.github
tests
.git
.github
.circleci
.devcontainer
*.tgz
log.txt
docker/Dockerfile.*
# Claude Flow generated files (must be excluded from Docker build)
.claude/
.claude-flow/
.swarm/
.hive-mind/
memory/
coordination/
claude-flow
.mcp.json
hive-mind-prompt-*.txt
# Python virtual environments and version managers
.venv/
venv/
**/.venv/
**/venv/
.python-version
.pyenv/
__pycache__/
**/__pycache__/
*.pyc
.mypy_cache/
.pytest_cache/
.ruff_cache/
**/pyvenv.cfg
# Common project exclusions
.vscode
*.pyo
*.pyd
.Python
env/
.pytest_cache
.coverage
htmlcov/
dist/
build/
*.egg-info/
.DS_Store
node_modules/
*.log
.env
.env.local

11
.env.example
View file

@ -1,6 +1,6 @@
# OpenAI
OPENAI_API_KEY = ""
OPENAI_API_BASE = ""
OPENAI_BASE_URL = ""
# Cohere
COHERE_API_KEY = ""
# OpenRouter
@ -20,3 +20,12 @@ REPLICATE_API_TOKEN = ""
ANTHROPIC_API_KEY = ""
# Infisical
INFISICAL_TOKEN = ""
# Novita AI
NOVITA_API_KEY = ""
# INFINITY
INFINITY_API_KEY = ""
# Development Configs
LITELLM_MASTER_KEY = "sk-1234"
DATABASE_URL = "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
STORE_MODEL_IN_DB = "True"

111
.gitguardian.yaml Normal file
View file

@ -0,0 +1,111 @@
version: 2
secret:
# Exclude files and paths by globbing
ignored_paths:
- "**/*.whl"
- "**/*.pyc"
- "**/__pycache__/**"
- "**/node_modules/**"
- "**/dist/**"
- "**/build/**"
- "**/.git/**"
- "**/venv/**"
- "**/.venv/**"
# Large data/metadata files that don't need scanning
- "**/model_prices_and_context_window*.json"
- "**/*_metadata/*.txt"
- "**/tokenizers/*.json"
- "**/tokenizers/*"
- "miniconda.sh"
# Build outputs and static assets
- "litellm/proxy/_experimental/out/**"
- "ui/litellm-dashboard/public/**"
- "**/swagger/*.js"
- "**/*.woff"
- "**/*.woff2"
- "**/*.avif"
- "**/*.webp"
# Test data files
- "**/tests/**/data_map.txt"
- "tests/**/*.txt"
# Documentation and other non-code files
- "docs/**"
- "**/*.md"
- "**/*.lock"
- "poetry.lock"
- "package-lock.json"
# Ignore security incidents with the SHA256 of the occurrence (false positives)
ignored_matches:
# === Current detected false positives (SHA-based) ===
# gcs_pub_sub_body - folder name, not a password
- name: GCS pub/sub test folder name
match: 75f377c456eede69e5f6e47399ccee6016a2a93cc5dd11db09cc5b1359ae569a
# os.environ/APORIA_API_KEY_1 - environment variable reference
- name: Environment variable reference APORIA_API_KEY_1
match: e2ddeb8b88eca97a402559a2be2117764e11c074d86159ef9ad2375dea188094
# os.environ/APORIA_API_KEY_2 - environment variable reference
- name: Environment variable reference APORIA_API_KEY_2
match: 09aa39a29e050b86603aa55138af1ff08fb86a4582aa965c1bd0672e1575e052
# oidc/circleci_v2/ - test authentication path, not a secret
- name: OIDC CircleCI test path
match: feb3475e1f89a65b7b7815ac4ec597e18a9ec1847742ad445c36ca617b536e15
# text-davinci-003 - OpenAI model identifier, not a secret
- name: OpenAI model identifier text-davinci-003
match: c489000cf6c7600cee0eefb80ad0965f82921cfb47ece880930eb7e7635cf1f1
# Base64 Basic Auth in test_pass_through_endpoints.py - test fixture, not a real secret
- name: Test Base64 Basic Auth header in pass_through_endpoints test
match: 61bac0491f395040617df7ef6d06029eac4d92a4457ac784978db80d97be1ae0
# PostgreSQL password "postgres" in CI configs - standard test database password
- name: Test PostgreSQL password in CI configurations
match: 6e0d657eb1f0fbc40cf0b8f3c3873ef627cc9cb7c4108d1c07d979c04bc8a4bb
# Bearer token in locustfile.py - test/example API key for load testing
- name: Test Bearer token in locustfile load test
match: 2a0abc2b0c3c1760a51ffcdf8d6b1d384cef69af740504b1cfa82dd70cdc7ff9
# Inkeep API key in docusaurus.config.js - public documentation site key
- name: Inkeep API key in documentation config
match: c366657791bfb5fc69045ec11d49452f09a0aebbc8648f94e2469b4025e29a75
# Langfuse credentials in test_completion.py - test credentials for integration test
- name: Langfuse test credentials in test_completion
match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
# Test password "sk-1234" in e2e test fixtures - test fixture, not a real secret
- name: Test password in e2e test fixtures
match: ce32b547202e209ec1dd50107b64be4cfcf2eb15c3b4f8e9dc611ef747af634f
# === Preventive patterns for test keys (pattern-based) ===
# Test API keys (124 instances across 45 files)
- name: Test API keys with sk-test prefix
match: sk-test-
# Mock API keys
- name: Mock API keys with sk-mock prefix
match: sk-mock-
# Fake API keys
- name: Fake API keys with sk-fake prefix
match: sk-fake-
# Generic test API key patterns
- name: Test API key patterns
match: test-api-key
- name: Short fake sk keys (1–9 digits only)
match: \bsk-\d{1,9}\b

38
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View file

@ -7,6 +7,16 @@ body:
attributes:
value: |
Thanks for taking the time to fill out this bug report!
**💡 Tip:** See our [Troubleshooting Guide](https://docs.litellm.ai/docs/troubleshoot) for what information to include.
- type: checkboxes
id: duplicate-check
attributes:
label: Check for existing issues
description: Please search to see if an issue already exists for the bug you encountered.
options:
- label: I have searched the existing issues and checked that my issue is not a duplicate.
required: true
- type: textarea
id: what-happened
attributes:
@ -16,6 +26,21 @@ body:
value: "A bug happened!"
validations:
required: true
- type: textarea
id: steps-to-reproduce
attributes:
label: Steps to Reproduce
description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
placeholder: |
1. config.yaml file/ .env file/ etc.
2. Run the following code...
3. Observe the error...
value: |
1.
2.
3.
validations:
required: true
- type: textarea
id: logs
attributes:
@ -23,13 +48,16 @@ body:
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
render: shell
- type: dropdown
id: ml-ops-team
id: component
attributes:
label: Are you a ML Ops Team?
description: This helps us prioritize your requests correctly
label: What part of LiteLLM is this about?
options:
- "No"
- "Yes"
- ''
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
- "Docs"
- "Other"
validations:
required: true
- type: input

27
.github/ISSUE_TEMPLATE/feature_request.yml vendored
View file

@ -7,6 +7,14 @@ body:
attributes:
value: |
Thanks for making LiteLLM better!
- type: checkboxes
id: duplicate-check
attributes:
label: Check for existing issues
description: Please search to see if an issue already exists for the feature you are requesting.
options:
- label: I have searched the existing issues and checked that my issue is not a duplicate.
required: true
- type: textarea
id: the-feature
attributes:
@ -23,10 +31,23 @@ body:
validations:
required: true
- type: dropdown
id: ml-ops-team
id: component
attributes:
label: Are you a ML Ops Team?
description: This helps us prioritize your requests correctly
label: What part of LiteLLM is this about?
options:
- ''
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
- "Docs"
- "Other"
validations:
required: true
- type: dropdown
id: hiring-interest
attributes:
label: LiteLLM is hiring a founding backend engineer, are you interested in joining us and shipping to all our users?
description: If yes, apply here - https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer
options:
- "No"
- "Yes"

37
.github/pull_request_template.md vendored
View file

@ -1,11 +1,32 @@
## Title
<!-- e.g. "Implement user authentication feature" -->
## Relevant issues
<!-- e.g. "Fixes #000" -->
## Pre-Submission checklist
**Please complete all items before asking a LiteLLM maintainer to review your PR**
- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
## CI (LiteLLM team)
> **CI status guideline:**
>
> - 50-55 passing tests: main is stable with minor issues.
> - 45-49 passing tests: acceptable but needs attention
> - <= 40 passing tests: unstable; be careful with your merges and assess the risk.
- [ ] **Branch creation CI run**
Link:
- [ ] **CI run for the last commit**
Link:
- [ ] **Merge / cherry-pick CI run**
Links:
## Type
<!-- Select the type of Pull Request -->
@ -19,11 +40,3 @@
✅ Test
## Changes
<!-- List of changes -->
## [REQUIRED] Testing - Attach a screenshot of any new tests passing locally
If UI changes, send a screenshot/GIF of working UI fixes
<!-- Test procedure -->

133
.github/scripts/scan_keywords.py vendored Normal file
View file

@ -0,0 +1,133 @@
#!/usr/bin/env python3
import json
import os
import sys
import urllib.request
import urllib.error
def read_event_payload() -> dict:
event_path = os.environ.get("GITHUB_EVENT_PATH")
if not event_path or not os.path.exists(event_path):
return {}
with open(event_path, "r", encoding="utf-8") as f:
return json.load(f)
def get_issue_text(event: dict) -> tuple[str, str, int, str, str]:
issue = event.get("issue") or {}
title = (issue.get("title") or "").strip()
body = (issue.get("body") or "").strip()
number = issue.get("number") or 0
html_url = issue.get("html_url") or ""
author = ((issue.get("user") or {}).get("login") or "").strip()
return title, body, number, html_url, author
def detect_keywords(text: str, keywords: list[str]) -> list[str]:
lowered = text.lower()
matches = []
for keyword in keywords:
k = keyword.strip().lower()
if not k:
continue
if k in lowered:
matches.append(keyword.strip())
# Deduplicate while preserving order
seen = set()
unique_matches = []
for m in matches:
if m not in seen:
unique_matches.append(m)
seen.add(m)
return unique_matches
def send_webhook(webhook_url: str, payload: dict) -> None:
if not webhook_url:
return
data = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(
webhook_url,
data=data,
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with urllib.request.urlopen(req, timeout=10) as resp:
resp.read()
except urllib.error.HTTPError as e:
print(f"Webhook HTTP error: {e.code} {e.reason}", file=sys.stderr)
except urllib.error.URLError as e:
print(f"Webhook URL error: {e.reason}", file=sys.stderr)
except Exception as e:
print(f"Webhook unexpected error: {e}", file=sys.stderr)
def _excerpt(text: str, max_len: int = 400) -> str:
if not text:
return ""
# Keep original formatting
if len(text) <= max_len:
return text
return text[: max_len - 1] + "…"
def main() -> int:
event = read_event_payload()
if not event:
print("::warning::No event payload found; exiting without labeling.")
return 0
# Read issue details
title, body, number, html_url, author = get_issue_text(event)
combined_text = f"{title}\n\n{body}".strip()
# Keywords from env or defaults
keywords_env = os.environ.get("KEYWORDS", "")
default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
matches = detect_keywords(combined_text, keywords)
found = bool(matches)
# Emit outputs
github_output = os.environ.get("GITHUB_OUTPUT")
if github_output:
with open(github_output, "a", encoding="utf-8") as fh:
fh.write(f"found={'true' if found else 'false'}\n")
fh.write(f"matches={','.join(matches)}\n")
# Optional webhook notification
webhook_url = os.environ.get("PROVIDER_ISSUE_WEBHOOK_URL", "").strip()
if found and webhook_url:
repo_full = (event.get("repository") or {}).get("full_name", "")
title_part = f"*{title}*" if title else "New issue"
author_part = f" by @{author}" if author else ""
body_preview = _excerpt(body)
preview_block = f"\n{body_preview}" if body_preview else ""
payload = {
"text": (
f"New issue 🚨\n"
f"{title_part}\n\n{preview_block}\n"
f"<{html_url}|View issue>\n"
f"Author: {author}"
)
}
send_webhook(webhook_url, payload)
# Print a short log line for Actions UI
if found:
print(f"Detected provider keywords: {', '.join(matches)}")
else:
print("No provider keywords detected.")
return 0
if __name__ == "__main__":
raise SystemExit(main())

35
.github/workflows/README.md vendored Normal file
View file

@ -0,0 +1,35 @@
# Simple PyPI Publishing
A GitHub workflow to manually publish LiteLLM packages to PyPI with a specified version.
## How to Use
1. Go to the **Actions** tab in the GitHub repository
2. Select **Simple PyPI Publish** from the workflow list
3. Click **Run workflow**
4. Enter the version to publish (e.g., `1.74.10`)
## What the Workflow Does
1. **Updates** the version in `pyproject.toml`
2. **Copies** the model prices backup file
3. **Builds** the Python package
4. **Publishes** to PyPI
## Prerequisites
Make sure the following secret is configured in the repository:
- `PYPI_PUBLISH_PASSWORD`: PyPI API token for authentication
## Example Usage
- Version: `1.74.11` → Publishes as v1.74.11
- Version: `1.74.10-hotfix1` → Publishes as v1.74.10-hotfix1
## Features
- ✅ Manual trigger with version input
- ✅ Automatic version updates in `pyproject.toml`
- ✅ Repository safety check (only runs on official repo)
- ✅ Clean package building and publishing
- ✅ Success confirmation with PyPI package link

60
.github/workflows/auto_update_price_and_context_window_file.py vendored
View file

@ -43,8 +43,8 @@ def write_to_file(file_path, data):
# Print an error message if writing to file fails
print("Error updating JSON file:", e)
# Update the existing models and add the missing models
def transform_remote_data(data):
# Update the existing models and add the missing models for OpenRouter
def transform_openrouter_data(data):
transformed = {}
for row in data:
# Add the fields 'max_tokens' and 'input_cost_per_token'
@ -81,6 +81,34 @@ def transform_remote_data(data):
return transformed
# Update the existing models and add the missing models for Vercel AI Gateway
def transform_vercel_ai_gateway_data(data):
transformed = {}
for row in data:
obj = {
"max_tokens": row["context_window"],
"input_cost_per_token": float(row["pricing"]["input"]),
"output_cost_per_token": float(row["pricing"]["output"]),
'max_output_tokens': row['max_tokens'],
'max_input_tokens': row["context_window"],
}
# Handle cache pricing if available
if "pricing" in row:
if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
mode = "embedding" if "embedding" in row["id"].lower() else "chat"
obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
return transformed
# Load local data from a specified file
def load_local_data(file_path):
@ -100,22 +128,32 @@ def load_local_data(file_path):
def main():
local_file_path = "model_prices_and_context_window.json" # Path to the local data file
url = "https://openrouter.ai/api/v1/models" # URL to fetch remote data
openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data
vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data
# Load local data from file
local_data = load_local_data(local_file_path)
# Fetch remote data asynchronously
remote_data = asyncio.run(fetch_data(url))
# Transform the fetched remote data
remote_data = transform_remote_data(remote_data)
# Fetch OpenRouter data
openrouter_data = asyncio.run(fetch_data(openrouter_url))
# Transform the fetched OpenRouter data
openrouter_data = transform_openrouter_data(openrouter_data)
# Fetch Vercel AI Gateway data
vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
# Transform the fetched Vercel AI Gateway data
vercel_data = transform_vercel_ai_gateway_data(vercel_data)
# Combine both datasets
all_remote_data = {**openrouter_data, **vercel_data}
# If both local and remote data are available, synchronize and save
if local_data and remote_data:
sync_local_data_with_remote(local_data, remote_data)
# If both local and openrouter data are available, synchronize and save
if local_data and all_remote_data:
sync_local_data_with_remote(local_data, all_remote_data)
write_to_file(local_file_path, local_data)
else:
print("Failed to fetch model data from either local file or URL.")
# Entry point of the script
if __name__ == "__main__":
main()
main()

29
.github/workflows/check_duplicate_issues.yml vendored Normal file
View file

@ -0,0 +1,29 @@
name: Check Duplicate Issues
on:
issues:
types: [opened, edited]
jobs:
check-duplicate:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Check for potential duplicates
uses: wow-actions/potential-duplicates@v1
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
label: potential-duplicate
threshold: 0.6
reaction: eyes
comment: |
**⚠️ Potential duplicate detected**
This issue appears similar to existing issue(s):
{{#issues}}
- [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
{{/issues}}
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.

43
.github/workflows/create_daily_staging_branch.yml vendored Normal file
View file

@ -0,0 +1,43 @@
name: Create Daily Staging Branch
on:
schedule:
- cron: '0 0,12 * * *' # Runs every 12 hours at midnight and noon UTC
workflow_dispatch: # Allow manual trigger
jobs:
create-staging-branch:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Create daily staging branch
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Configure Git user
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
# Generate branch name with MM_DD_YYYY format
BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')"
echo "Creating branch: $BRANCH_NAME"
# Fetch all branches
git fetch --all
# Check if the branch already exists
if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
echo "Branch $BRANCH_NAME already exists. Skipping creation."
else
echo "Creating new branch: $BRANCH_NAME"
# Create the new branch from main
git checkout -b $BRANCH_NAME origin/main
# Push the new branch
git push origin $BRANCH_NAME
echo "Successfully created and pushed branch: $BRANCH_NAME"
fi

134
.github/workflows/ghcr_deploy.yml vendored
View file

@ -5,8 +5,9 @@ on:
inputs:
tag:
description: "The tag version you want to build"
required: true
release_type:
description: "The release type you want to build. Can be 'latest', 'stable', 'dev'"
description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
type: string
default: "latest"
commit_hash:
@ -73,14 +74,20 @@ jobs:
push: true
file: ./litellm-js/spend-logs/Dockerfile
tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }}
-
name: Build and push litellm-non_root image
uses: docker/build-push-action@v5
with:
context: .
push: true
file: ./docker/Dockerfile.non_root
tags: litellm/litellm-non_root:${{ github.event.inputs.tag || 'latest' }}
build-and-push-image:
runs-on: ubuntu-latest
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
permissions:
contents: read
packages: write
#
steps:
- name: Checkout repository
uses: actions/checkout@v4
@ -112,10 +119,58 @@ jobs:
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest'
tags: |
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }},
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
labels: ${{ steps.meta.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-image-ee:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for EE Dockerfile
id: meta-ee
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
# Configure multi platform Docker builds
- name: Set up QEMU
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
- name: Build and push EE Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: .
file: Dockerfile
push: true
tags: |
${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-ee.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-image-database:
runs-on: ubuntu-latest
permissions:
@ -151,8 +206,12 @@ jobs:
context: .
file: ./docker/Dockerfile.database
push: true
tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
labels: ${{ steps.meta-database.outputs.labels }}
tags: |
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-database.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-image-non_root:
@ -190,7 +249,11 @@ jobs:
context: .
file: ./docker/Dockerfile.non_root
push: true
tags: ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
tags: |
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-non_root.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
@ -229,7 +292,11 @@ jobs:
context: .
file: ./litellm-js/spend-logs/Dockerfile
push: true
tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
tags: |
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-helm-chart:
@ -253,44 +320,37 @@ jobs:
run: |
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
- name: Get LiteLLM Latest Tag
id: current_app_tag
# Sync Helm chart version with LiteLLM release version (1-1 versioning)
# This allows users to easily map Helm chart versions to LiteLLM versions
# See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/
- name: Calculate chart and app versions
id: chart_version
shell: bash
run: |
LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0)
if [ -z "${LATEST_TAG}" ]; then
echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT
else
echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT
INPUT_TAG="${{ github.event.inputs.tag }}"
RELEASE_TYPE="${{ github.event.inputs.release_type }}"
# Chart version = LiteLLM version without 'v' prefix (Helm semver convention)
# v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1
CHART_VERSION="${INPUT_TAG#v}"
# Add suffix for 'latest' releases (rc already has suffix in tag)
if [ "$RELEASE_TYPE" = "latest" ]; then
CHART_VERSION="${CHART_VERSION}-latest"
fi
- name: Get last published chart version
id: current_version
shell: bash
run: |
CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
if [ -z "${CHART_LIST}" ]; then
echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
else
printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
fi
env:
HELM_EXPERIMENTAL_OCI: '1'
# App version = Docker tag (keeps 'v' prefix to match Docker image tags)
APP_VERSION="${INPUT_TAG}"
# Automatically update the helm chart version one "patch" level
- name: Bump release version
id: bump_version
uses: christian-draeger/increment-semantic-version@1.1.0
with:
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
version-fragment: 'bug'
echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
- uses: ./.github/actions/helm-oci-chart-releaser
with:
name: ${{ env.CHART_NAME }}
repository: ${{ env.REPO_OWNER }}
tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
app_version: ${{ steps.current_app_tag.outputs.latest_tag }}
tag: ${{ steps.chart_version.outputs.version }}
app_version: ${{ steps.chart_version.outputs.app_version }}
path: deploy/charts/${{ env.CHART_NAME }}
registry: ${{ env.REGISTRY }}
registry_username: ${{ github.actor }}

42
.github/workflows/ghcr_helm_deploy.yml vendored
View file

@ -1,10 +1,12 @@
# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
# Standalone workflow to publish LiteLLM Helm Chart
# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release
name: Build, Publish LiteLLM Helm Chart. New Release
on:
workflow_dispatch:
inputs:
chartVersion:
description: "Update the helm chart's version to this"
tag:
description: "LiteLLM version tag (e.g., v1.81.0)"
required: true
# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
@ -31,24 +33,22 @@ jobs:
run: |
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
- name: Get LiteLLM Latest Tag
id: current_app_tag
uses: WyriHaximus/github-action-get-previous-tag@v1.3.0
- name: Get last published chart version
id: current_version
# Sync Helm chart version with LiteLLM release version (1-1 versioning)
- name: Calculate chart and app versions
id: chart_version
shell: bash
run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
env:
HELM_EXPERIMENTAL_OCI: '1'
run: |
INPUT_TAG="${{ github.event.inputs.tag }}"
# Automatically update the helm chart version one "patch" level
- name: Bump release version
id: bump_version
uses: christian-draeger/increment-semantic-version@1.1.0
with:
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
version-fragment: 'bug'
# Chart version = LiteLLM version without 'v' prefix
# v1.81.0 -> 1.81.0
CHART_VERSION="${INPUT_TAG#v}"
# App version = Docker tag (keeps 'v' prefix)
APP_VERSION="${INPUT_TAG}"
echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
- name: Lint helm chart
run: helm lint deploy/charts/litellm-helm
@ -57,8 +57,8 @@ jobs:
with:
name: litellm-helm
repository: ${{ env.REPO_OWNER }}
tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
tag: ${{ steps.chart_version.outputs.version }}
app_version: ${{ steps.chart_version.outputs.app_version }}
path: deploy/charts/litellm-helm
registry: ${{ env.REGISTRY }}
registry_username: ${{ github.actor }}

27
.github/workflows/helm_unit_test.yml vendored Normal file
View file

@ -0,0 +1,27 @@
name: Helm unit test
on:
pull_request:
push:
branches:
- main
jobs:
unit-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Set up Helm 3.11.1
uses: azure/setup-helm@v1
with:
version: '3.11.1'
- name: Install Helm Unit Test Plugin
run: |
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
- name: Run unit tests
run:
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm

48
.github/workflows/interpret_load_test.py vendored
View file

@ -52,7 +52,43 @@ def interpret_results(csv_file):
return markdown_table
def _get_docker_run_command_stable_release(release_version):
return f"""
\n\n
## Docker Run LiteLLM Proxy
```
docker run \\
-e STORE_MODEL_IN_DB=True \\
-p 4000:4000 \\
ghcr.io/berriai/litellm:litellm_stable_release_branch-{release_version}
```
"""
def _get_docker_run_command(release_version):
return f"""
\n\n
## Docker Run LiteLLM Proxy
```
docker run \\
-e STORE_MODEL_IN_DB=True \\
-p 4000:4000 \\
ghcr.io/berriai/litellm:main-{release_version}
```
"""
def get_docker_run_command(release_version):
if "stable" in release_version:
return _get_docker_run_command_stable_release(release_version)
else:
return _get_docker_run_command(release_version)
if __name__ == "__main__":
return
csv_file = "load_test_stats.csv" # Change this to the path of your CSV file
markdown_table = interpret_results(csv_file)
@ -79,17 +115,7 @@ if __name__ == "__main__":
start_index = latest_release.body.find("Load Test LiteLLM Proxy Results")
existing_release_body = latest_release.body[:start_index]
docker_run_command = f"""
\n\n
## Docker Run LiteLLM Proxy
```
docker run \\
-e STORE_MODEL_IN_DB=True \\
-p 4000:4000 \\
ghcr.io/berriai/litellm:main-{release_version}
```
"""
docker_run_command = get_docker_run_command(release_version)
print("docker run command: ", docker_run_command)
new_release_body = (

64
.github/workflows/issue-keyword-labeler.yml vendored Normal file
View file

@ -0,0 +1,64 @@
name: Issue Keyword Labeler
on:
issues:
types:
- opened
jobs:
scan-and-label:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Scan for provider keywords
id: scan
env:
PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic,gemini,cohere,mistral,groq,ollama,deepseek
run: python3 .github/scripts/scan_keywords.py
- name: Ensure label exists
if: steps.scan.outputs.found == 'true'
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'llm translation';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'c1ff72',
description: 'Issues related to LLM provider translation/mapping'
});
} else {
throw error;
}
}
- name: Add label to the issue
if: steps.scan.outputs.found == 'true'
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: ['llm translation']
});

116
.github/workflows/label-component.yml vendored Normal file
View file

@ -0,0 +1,116 @@
name: Label Component Issues
on:
issues:
types:
- opened
jobs:
add-component-label:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Add component labels
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const body = context.payload.issue.body;
if (!body) return;
// Define component mappings with regex patterns that handle flexible whitespace
const components = [
{
pattern: /What part of LiteLLM is this about\?\s*SDK \(litellm Python package\)/,
label: 'sdk',
color: '0E7C86',
description: 'Issues related to the litellm Python SDK'
},
{
pattern: /What part of LiteLLM is this about\?\s*Proxy/,
label: 'proxy',
color: '5319E7',
description: 'Issues related to the LiteLLM Proxy'
},
{
pattern: /What part of LiteLLM is this about\?\s*UI Dashboard/,
label: 'ui-dashboard',
color: 'D876E3',
description: 'Issues related to the LiteLLM UI Dashboard'
},
{
pattern: /What part of LiteLLM is this about\?\s*Docs/,
label: 'docs',
color: 'FBCA04',
description: 'Issues related to LiteLLM documentation'
}
];
// Find matching component
for (const component of components) {
if (component.pattern.test(body)) {
// Ensure label exists
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: component.label
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: component.label,
color: component.color,
description: component.description
});
}
}
// Add label to issue
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [component.label]
});
break;
}
}
// Check for 'claude code' keyword (can be applied alongside component labels)
if (/claude code/i.test(body)) {
const claudeLabel = {
name: 'claude code',
color: '7c3aed',
description: 'Issues related to Claude Code usage'
};
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: claudeLabel.name
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: claudeLabel.name,
color: claudeLabel.color,
description: claudeLabel.description
});
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [claudeLabel.name]
});
}

17
.github/workflows/label-mlops.yml vendored
View file

@ -1,17 +0,0 @@
name: Label ML Ops Team Issues
on:
issues:
types:
- opened
jobs:
add-mlops-label:
runs-on: ubuntu-latest
steps:
- name: Check if ML Ops Team is selected
uses: actions-ecosystem/action-add-labels@v1
if: contains(github.event.issue.body, '### Are you a ML Ops Team?') && contains(github.event.issue.body, 'Yes')
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
labels: "mlops user request"

89
.github/workflows/llm-translation-testing.yml vendored Normal file
View file

@ -0,0 +1,89 @@
name: LLM Translation Tests
on:
workflow_dispatch:
inputs:
release_candidate_tag:
description: 'Release candidate tag/version'
required: true
type: string
push:
tags:
- 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1
jobs:
run-llm-translation-tests:
runs-on: ubuntu-latest
timeout-minutes: 90
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: latest
virtualenvs-create: true
virtualenvs-in-project: true
- name: Cache Poetry dependencies
uses: actions/cache@v3
with:
path: |
~/.cache/pypoetry
.venv
key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
restore-keys: |
${{ runner.os }}-poetry-
- name: Install dependencies
run: |
poetry install --with dev
poetry run pip install pytest-xdist pytest-timeout
- name: Create test results directory
run: mkdir -p test-results
- name: Run LLM Translation Tests
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
# Add other API keys as needed
run: |
python .github/workflows/run_llm_translation_tests.py \
--tag "${{ github.event.inputs.release_candidate_tag || github.ref_name }}" \
--commit "${{ github.sha }}" \
|| true # Continue even if tests fail
- name: Display test summary
if: always()
run: |
if [ -f "test-results/llm_translation_report.md" ]; then
echo "Test report generated successfully!"
echo "Artifact will contain:"
echo "- test-results/junit.xml (JUnit XML results)"
echo "- test-results/llm_translation_report.md (Beautiful markdown report)"
else
echo "Warning: Test report was not generated"
fi
- name: Upload test artifacts
uses: actions/upload-artifact@v4
if: always()
with:
name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
path: test-results/
retention-days: 30

2
.github/workflows/locustfile.py vendored
View file

@ -8,7 +8,7 @@ class MyUser(HttpUser):
def chat_completion(self):
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer sk-ZoHqrLIs2-5PzJrqBaviAA",
"Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg",
# Include any additional headers you may need for authentication, etc.
}

207
.github/workflows/publish-migrations.yml vendored Normal file
View file

@ -0,0 +1,207 @@
name: Publish Prisma Migrations
permissions:
contents: write
pull-requests: write
on:
push:
paths:
- 'schema.prisma' # Check root schema.prisma
branches:
- main
jobs:
publish-migrations:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
services:
postgres:
image: postgres:14
env:
POSTGRES_DB: temp_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
# Add shadow database service
postgres_shadow:
image: postgres:14
env:
POSTGRES_DB: shadow_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
ports:
- 5433:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Install Dependencies
run: |
pip install prisma
pip install python-dotenv
- name: Generate Initial Migration if None Exists
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
run: |
mkdir -p deploy/migrations
echo 'provider = "postgresql"' > deploy/migrations/migration_lock.toml
if [ -z "$(ls -A deploy/migrations/2* 2>/dev/null)" ]; then
echo "No existing migrations found, creating baseline..."
VERSION=$(date +%Y%m%d%H%M%S)
mkdir -p deploy/migrations/${VERSION}_initial
echo "Generating initial migration..."
# Save raw output for debugging
prisma migrate diff \
--from-empty \
--to-schema-datamodel schema.prisma \
--shadow-database-url "${SHADOW_DATABASE_URL}" \
--script > deploy/migrations/${VERSION}_initial/raw_migration.sql
echo "Raw migration file content:"
cat deploy/migrations/${VERSION}_initial/raw_migration.sql
echo "Cleaning migration file..."
# Clean the file
sed '/^Installing/d' deploy/migrations/${VERSION}_initial/raw_migration.sql > deploy/migrations/${VERSION}_initial/migration.sql
# Verify the migration file
if [ ! -s deploy/migrations/${VERSION}_initial/migration.sql ]; then
echo "ERROR: Migration file is empty after cleaning"
echo "Original content was:"
cat deploy/migrations/${VERSION}_initial/raw_migration.sql
exit 1
fi
echo "Final migration file content:"
cat deploy/migrations/${VERSION}_initial/migration.sql
# Verify it starts with SQL
if ! head -n 1 deploy/migrations/${VERSION}_initial/migration.sql | grep -q "^--\|^CREATE\|^ALTER"; then
echo "ERROR: Migration file does not start with SQL command or comment"
echo "First line is:"
head -n 1 deploy/migrations/${VERSION}_initial/migration.sql
echo "Full content is:"
cat deploy/migrations/${VERSION}_initial/migration.sql
exit 1
fi
echo "Initial migration generated at $(date -u)" > deploy/migrations/${VERSION}_initial/README.md
fi
- name: Compare and Generate Migration
if: success()
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
run: |
# Create temporary migration workspace
mkdir -p temp_migrations
# Copy existing migrations (will not fail if directory is empty)
cp -r deploy/migrations/* temp_migrations/ 2>/dev/null || true
VERSION=$(date +%Y%m%d%H%M%S)
# Generate diff against existing migrations or empty state
prisma migrate diff \
--from-migrations temp_migrations \
--to-schema-datamodel schema.prisma \
--shadow-database-url "${SHADOW_DATABASE_URL}" \
--script > temp_migrations/migration_${VERSION}.sql
# Check if there are actual changes
if [ -s temp_migrations/migration_${VERSION}.sql ]; then
echo "Changes detected, creating new migration"
mkdir -p deploy/migrations/${VERSION}_schema_update
mv temp_migrations/migration_${VERSION}.sql deploy/migrations/${VERSION}_schema_update/migration.sql
echo "Migration generated at $(date -u)" > deploy/migrations/${VERSION}_schema_update/README.md
else
echo "No schema changes detected"
exit 0
fi
- name: Verify Migration
if: success()
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
run: |
# Create test database
psql "${SHADOW_DATABASE_URL}" -c 'CREATE DATABASE migration_test;'
# Apply all migrations in order to verify
for migration in deploy/migrations/*/migration.sql; do
echo "Applying migration: $migration"
psql "${SHADOW_DATABASE_URL}" -f $migration
done
# Add this step before create-pull-request to debug permissions
- name: Check Token Permissions
run: |
echo "Checking token permissions..."
curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/BerriAI/litellm/collaborators
echo "\nChecking if token can create PRs..."
curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/BerriAI/litellm
# Add this debug step before git push
- name: Debug Changed Files
run: |
echo "Files staged for commit:"
git diff --name-status --staged
echo "\nAll changed files:"
git status
- name: Create Pull Request
if: success()
uses: peter-evans/create-pull-request@v5
with:
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: "chore: update prisma migrations"
title: "Update Prisma Migrations"
body: |
Auto-generated migration based on schema.prisma changes.
Generated files:
- deploy/migrations/${VERSION}_schema_update/migration.sql
- deploy/migrations/${VERSION}_schema_update/README.md
branch: feat/prisma-migration-${{ env.VERSION }}
base: main
delete-branch: true
- name: Generate and Save Migrations
run: |
# Only add migration files
git add deploy/migrations/
git status # Debug what's being committed
git commit -m "chore: update prisma migrations"

439
.github/workflows/run_llm_translation_tests.py vendored Executable file
View file

@ -0,0 +1,439 @@
#!/usr/bin/env python3
"""
Run LLM Translation Tests and Generate Beautiful Markdown Report
This script runs the LLM translation tests and generates a comprehensive
markdown report with provider-specific breakdowns and test statistics.
"""
import os
import sys
import subprocess
import xml.etree.ElementTree as ET
from collections import defaultdict
from datetime import datetime
from pathlib import Path
import json
from typing import Dict, List, Tuple, Optional
# ANSI color codes for terminal output
class Colors:
GREEN = '\033[92m'
RED = '\033[91m'
YELLOW = '\033[93m'
BLUE = '\033[94m'
PURPLE = '\033[95m'
CYAN = '\033[96m'
RESET = '\033[0m'
BOLD = '\033[1m'
def print_colored(message: str, color: str = Colors.RESET):
"""Print colored message to terminal"""
print(f"{color}{message}{Colors.RESET}")
def get_provider_from_test_file(test_file: str) -> str:
"""Map test file names to provider names"""
provider_mapping = {
'test_anthropic': 'Anthropic',
'test_azure': 'Azure',
'test_bedrock': 'AWS Bedrock',
'test_openai': 'OpenAI',
'test_vertex': 'Google Vertex AI',
'test_gemini': 'Google Vertex AI',
'test_cohere': 'Cohere',
'test_databricks': 'Databricks',
'test_groq': 'Groq',
'test_together': 'Together AI',
'test_mistral': 'Mistral',
'test_deepseek': 'DeepSeek',
'test_replicate': 'Replicate',
'test_huggingface': 'HuggingFace',
'test_fireworks': 'Fireworks AI',
'test_perplexity': 'Perplexity',
'test_cloudflare': 'Cloudflare',
'test_voyage': 'Voyage AI',
'test_xai': 'xAI',
'test_nvidia': 'NVIDIA',
'test_watsonx': 'IBM watsonx',
'test_azure_ai': 'Azure AI',
'test_snowflake': 'Snowflake',
'test_infinity': 'Infinity',
'test_jina': 'Jina AI',
'test_deepgram': 'Deepgram',
'test_clarifai': 'Clarifai',
'test_triton': 'Triton',
}
for key, provider in provider_mapping.items():
if key in test_file:
return provider
# For cross-provider test files
if any(name in test_file for name in ['test_optional_params', 'test_prompt_factory',
'test_router', 'test_text_completion']):
return f'Cross-Provider Tests ({test_file})'
return 'Other Tests'
def format_duration(seconds: float) -> str:
"""Format duration in human-readable format"""
if seconds < 60:
return f"{seconds:.2f}s"
elif seconds < 3600:
minutes = int(seconds // 60)
secs = seconds % 60
return f"{minutes}m {secs:.0f}s"
else:
hours = int(seconds // 3600)
minutes = int((seconds % 3600) // 60)
return f"{hours}h {minutes}m"
def generate_markdown_report(junit_xml_path: str, output_path: str, tag: str = None, commit: str = None):
"""Generate a beautiful markdown report from JUnit XML"""
try:
tree = ET.parse(junit_xml_path)
root = tree.getroot()
# Handle both testsuite and testsuites root
if root.tag == 'testsuites':
suites = root.findall('testsuite')
else:
suites = [root]
# Overall statistics
total_tests = 0
total_failures = 0
total_errors = 0
total_skipped = 0
total_time = 0.0
# Provider breakdown
provider_stats = defaultdict(lambda: {'passed': 0, 'failed': 0, 'skipped': 0, 'errors': 0, 'time': 0.0})
provider_tests = defaultdict(list)
for suite in suites:
total_tests += int(suite.get('tests', 0))
total_failures += int(suite.get('failures', 0))
total_errors += int(suite.get('errors', 0))
total_skipped += int(suite.get('skipped', 0))
total_time += float(suite.get('time', 0))
for testcase in suite.findall('testcase'):
classname = testcase.get('classname', '')
test_name = testcase.get('name', '')
test_time = float(testcase.get('time', 0))
# Extract test file name from classname
if '.' in classname:
parts = classname.split('.')
test_file = parts[-2] if len(parts) > 1 else 'unknown'
else:
test_file = 'unknown'
provider = get_provider_from_test_file(test_file)
provider_stats[provider]['time'] += test_time
# Check test status
if testcase.find('failure') is not None:
provider_stats[provider]['failed'] += 1
failure = testcase.find('failure')
failure_msg = failure.get('message', '') if failure is not None else ''
provider_tests[provider].append({
'name': test_name,
'status': 'FAILED',
'time': test_time,
'message': failure_msg
})
elif testcase.find('error') is not None:
provider_stats[provider]['errors'] += 1
error = testcase.find('error')
error_msg = error.get('message', '') if error is not None else ''
provider_tests[provider].append({
'name': test_name,
'status': 'ERROR',
'time': test_time,
'message': error_msg
})
elif testcase.find('skipped') is not None:
provider_stats[provider]['skipped'] += 1
skip = testcase.find('skipped')
skip_msg = skip.get('message', '') if skip is not None else ''
provider_tests[provider].append({
'name': test_name,
'status': 'SKIPPED',
'time': test_time,
'message': skip_msg
})
else:
provider_stats[provider]['passed'] += 1
provider_tests[provider].append({
'name': test_name,
'status': 'PASSED',
'time': test_time,
'message': ''
})
passed = total_tests - total_failures - total_errors - total_skipped
# Generate the markdown report
with open(output_path, 'w') as f:
# Header
f.write("# LLM Translation Test Results\n\n")
# Metadata table
f.write("## Test Run Information\n\n")
f.write("| Field | Value |\n")
f.write("|-------|-------|\n")
f.write(f"| **Tag** | `{tag or 'N/A'}` |\n")
f.write(f"| **Date** | {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')} |\n")
f.write(f"| **Commit** | `{commit or 'N/A'}` |\n")
f.write(f"| **Duration** | {format_duration(total_time)} |\n")
f.write("\n")
# Overall statistics with visual elements
f.write("## Overall Statistics\n\n")
# Summary box
f.write("```\n")
f.write(f"Total Tests: {total_tests}\n")
f.write(f"├── Passed: {passed:>4} ({(passed/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
f.write(f"├── Failed: {total_failures:>4} ({(total_failures/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
f.write(f"├── Errors: {total_errors:>4} ({(total_errors/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
f.write(f"└── Skipped: {total_skipped:>4} ({(total_skipped/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
f.write("```\n\n")
# Provider summary table
f.write("## Results by Provider\n\n")
f.write("| Provider | Total | Pass | Fail | Error | Skip | Pass Rate | Duration |\n")
f.write("|----------|-------|------|------|-------|------|-----------|----------|")
# Sort providers: specific providers first, then cross-provider tests
sorted_providers = []
cross_provider = []
for p in sorted(provider_stats.keys()):
if 'Cross-Provider' in p or p == 'Other Tests':
cross_provider.append(p)
else:
sorted_providers.append(p)
all_providers = sorted_providers + cross_provider
for provider in all_providers:
stats = provider_stats[provider]
total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
pass_rate = (stats['passed'] / total * 100) if total > 0 else 0
f.write(f"\n| {provider} | {total} | {stats['passed']} | {stats['failed']} | ")
f.write(f"{stats['errors']} | {stats['skipped']} | {pass_rate:.1f}% | ")
f.write(f"{format_duration(stats['time'])} |")
# Detailed test results by provider
f.write("\n\n## Detailed Test Results\n\n")
for provider in sorted_providers:
if provider_tests[provider]:
stats = provider_stats[provider]
total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
f.write(f"### {provider}\n\n")
f.write(f"**Summary:** {stats['passed']}/{total} passed ")
f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%) ")
f.write(f"in {format_duration(stats['time'])}\n\n")
# Group tests by status
tests_by_status = defaultdict(list)
for test in provider_tests[provider]:
tests_by_status[test['status']].append(test)
# Show failed tests first (if any)
if tests_by_status['FAILED']:
f.write("<details>\n<summary>Failed Tests</summary>\n\n")
for test in tests_by_status['FAILED']:
f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
if test['message']:
# Truncate long error messages
msg = test['message'][:200] + '...' if len(test['message']) > 200 else test['message']
f.write(f" > {msg}\n")
f.write("\n</details>\n\n")
# Show errors (if any)
if tests_by_status['ERROR']:
f.write("<details>\n<summary>Error Tests</summary>\n\n")
for test in tests_by_status['ERROR']:
f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
f.write("\n</details>\n\n")
# Show passed tests in collapsible section
if tests_by_status['PASSED']:
f.write("<details>\n<summary>Passed Tests</summary>\n\n")
for test in tests_by_status['PASSED']:
f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
f.write("\n</details>\n\n")
# Show skipped tests (if any)
if tests_by_status['SKIPPED']:
f.write("<details>\n<summary>Skipped Tests</summary>\n\n")
for test in tests_by_status['SKIPPED']:
f.write(f"- `{test['name']}`\n")
f.write("\n</details>\n\n")
# Cross-provider tests in a separate section
if cross_provider:
f.write("### Cross-Provider Tests\n\n")
for provider in cross_provider:
if provider_tests[provider]:
stats = provider_stats[provider]
total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
f.write(f"#### {provider}\n\n")
f.write(f"**Summary:** {stats['passed']}/{total} passed ")
f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%)\n\n")
# For cross-provider tests, just show counts
f.write(f"- Passed: {stats['passed']}\n")
if stats['failed'] > 0:
f.write(f"- Failed: {stats['failed']}\n")
if stats['errors'] > 0:
f.write(f"- Errors: {stats['errors']}\n")
if stats['skipped'] > 0:
f.write(f"- Skipped: {stats['skipped']}\n")
f.write("\n")
print_colored(f"Report generated: {output_path}", Colors.GREEN)
except Exception as e:
print_colored(f"Error generating report: {e}", Colors.RED)
raise
def run_tests(test_path: str = "tests/llm_translation/",
junit_xml: str = "test-results/junit.xml",
report_path: str = "test-results/llm_translation_report.md",
tag: str = None,
commit: str = None) -> int:
"""Run the LLM translation tests and generate report"""
# Create test results directory
os.makedirs(os.path.dirname(junit_xml), exist_ok=True)
print_colored("Starting LLM Translation Tests", Colors.BOLD + Colors.BLUE)
print_colored(f"Test directory: {test_path}", Colors.CYAN)
print_colored(f"Output: {junit_xml}", Colors.CYAN)
print()
# Run pytest
cmd = [
"poetry", "run", "pytest", test_path,
f"--junitxml={junit_xml}",
"-v",
"--tb=short",
"--maxfail=500",
"-n", "auto"
]
# Add timeout if pytest-timeout is installed
try:
subprocess.run(["poetry", "run", "python", "-c", "import pytest_timeout"],
capture_output=True, check=True)
cmd.extend(["--timeout=300"])
except:
print_colored("Warning: pytest-timeout not installed, skipping timeout option", Colors.YELLOW)
print_colored("Running pytest with command:", Colors.YELLOW)
print(f" {' '.join(cmd)}")
print()
# Run the tests
result = subprocess.run(cmd, capture_output=False)
# Generate the report regardless of test outcome
if os.path.exists(junit_xml):
print()
print_colored("Generating test report...", Colors.BLUE)
generate_markdown_report(junit_xml, report_path, tag, commit)
# Print summary to console
print()
print_colored("Test Summary:", Colors.BOLD + Colors.PURPLE)
# Parse XML for quick summary
tree = ET.parse(junit_xml)
root = tree.getroot()
if root.tag == 'testsuites':
suites = root.findall('testsuite')
else:
suites = [root]
total = sum(int(s.get('tests', 0)) for s in suites)
failures = sum(int(s.get('failures', 0)) for s in suites)
errors = sum(int(s.get('errors', 0)) for s in suites)
skipped = sum(int(s.get('skipped', 0)) for s in suites)
passed = total - failures - errors - skipped
print(f" Total: {total}")
print_colored(f" Passed: {passed}", Colors.GREEN)
if failures > 0:
print_colored(f" Failed: {failures}", Colors.RED)
if errors > 0:
print_colored(f" Errors: {errors}", Colors.RED)
if skipped > 0:
print_colored(f" Skipped: {skipped}", Colors.YELLOW)
if total > 0:
pass_rate = (passed / total) * 100
color = Colors.GREEN if pass_rate >= 80 else Colors.YELLOW if pass_rate >= 60 else Colors.RED
print_colored(f" Pass Rate: {pass_rate:.1f}%", color)
else:
print_colored("No test results found!", Colors.RED)
print()
print_colored("Test run complete!", Colors.BOLD + Colors.GREEN)
return result.returncode
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Run LLM Translation Tests")
parser.add_argument("--test-path", default="tests/llm_translation/",
help="Path to test directory")
parser.add_argument("--junit-xml", default="test-results/junit.xml",
help="Path for JUnit XML output")
parser.add_argument("--report", default="test-results/llm_translation_report.md",
help="Path for markdown report")
parser.add_argument("--tag", help="Git tag or version")
parser.add_argument("--commit", help="Git commit SHA")
args = parser.parse_args()
# Get git info if not provided
if not args.commit:
try:
result = subprocess.run(["git", "rev-parse", "HEAD"],
capture_output=True, text=True)
if result.returncode == 0:
args.commit = result.stdout.strip()
except:
pass
if not args.tag:
try:
result = subprocess.run(["git", "describe", "--tags", "--abbrev=0"],
capture_output=True, text=True)
if result.returncode == 0:
args.tag = result.stdout.strip()
except:
pass
exit_code = run_tests(
test_path=args.test_path,
junit_xml=args.junit_xml,
report_path=args.report,
tag=args.tag,
commit=args.commit
)
sys.exit(exit_code)

67
.github/workflows/simple_pypi_publish.yml vendored Normal file
View file

@ -0,0 +1,67 @@
name: Simple PyPI Publish
on:
workflow_dispatch:
inputs:
version:
description: 'Version to publish (e.g., 1.74.10)'
required: true
type: string
env:
TWINE_USERNAME: __token__
jobs:
publish:
runs-on: ubuntu-latest
if: github.repository == 'BerriAI/litellm'
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install toml build wheel twine
- name: Update version in pyproject.toml
run: |
python -c "
import toml
with open('pyproject.toml', 'r') as f:
data = toml.load(f)
data['tool']['poetry']['version'] = '${{ github.event.inputs.version }}'
with open('pyproject.toml', 'w') as f:
toml.dump(data, f)
print(f'Updated version to ${{ github.event.inputs.version }}')
"
- name: Copy model prices file
run: |
cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
- name: Build package
run: |
rm -rf build dist
python -m build
- name: Publish to PyPI
env:
TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
run: |
twine upload dist/*
- name: Output success
run: |
echo "✅ Successfully published litellm v${{ github.event.inputs.version }} to PyPI"
echo "📦 Package: https://pypi.org/project/litellm/${{ github.event.inputs.version }}/"

76
.github/workflows/test-linting.yml vendored Normal file
View file

@ -0,0 +1,76 @@
name: LiteLLM Linting
on:
pull_request:
branches: [ main ]
jobs:
lint:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
clean: true
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Clean Python cache
run: |
find . -type d -name "__pycache__" -exec rm -rf {} + || true
find . -name "*.pyc" -delete || true
- name: Install dependencies
run: |
poetry lock
poetry install --with dev
poetry run pip install openai==1.100.1
- name: Run Black formatting
run: |
cd litellm
poetry run black .
cd ..
- name: Debug - Check file state
run: |
echo "Current branch:"
git branch --show-current
echo "Last 3 commits:"
git log --oneline -3
echo "File content around line 43:"
head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
- name: Run Ruff linting
run: |
cd litellm
poetry run ruff check .
cd ..
- name: Print OpenAI version
run: |
poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Run MyPy type checking
run: |
cd litellm
poetry run mypy .
cd ..
- name: Check for circular imports
run: |
cd litellm
poetry run python ../tests/documentation_tests/test_circular_imports.py
cd ..
- name: Check import safety
run: |
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)

46
.github/workflows/test-litellm.yml vendored Normal file
View file

@ -0,0 +1,46 @@
name: LiteLLM Mock Tests (folder - tests/test_litellm)
on:
pull_request:
branches: [ main ]
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- name: Thank You Message
run: |
echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Install dependencies
run: |
poetry lock
poetry install --with dev,proxy-dev --extras "proxy semantic-router"
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
poetry run pip install "google-genai==1.22.0"
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.22"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
poetry run pip install -e .
cd ..
- name: Run tests
run: |
poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

49
.github/workflows/test-mcp.yml vendored Normal file
View file

@ -0,0 +1,49 @@
name: LiteLLM MCP Tests (folder - tests/mcp_tests)
on:
pull_request:
branches: [ main ]
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- name: Thank You Message
run: |
echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Install dependencies
run: |
poetry lock
poetry install --with dev,proxy-dev --extras "proxy semantic-router"
poetry run pip install "pytest==7.3.1"
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install "pytest-cov==5.0.0"
poetry run pip install "pytest-asyncio==0.21.1"
poetry run pip install "respx==0.22.0"
poetry run pip install "pydantic==2.11.0"
poetry run pip install "mcp==1.25.0"
poetry run pip install pytest-xdist
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
python -m pip install -e .
cd ..
- name: Run MCP tests
run: |
poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5

15
.github/workflows/test-model-map.yaml vendored Normal file
View file

@ -0,0 +1,15 @@
name: Validate model_prices_and_context_window.json
on:
pull_request:
branches: [ main ]
jobs:
validate-model-prices-json:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Validate model_prices_and_context_window.json
run: |
jq empty model_prices_and_context_window.json

36
.gitignore vendored
View file

@ -1,4 +1,6 @@
.python-version
.venv
.venv_policy_test
.env
.newenv
newenv/*
@ -58,9 +60,6 @@ litellm/proxy/_super_secret_config.yaml
litellm/proxy/myenv/bin/activate
litellm/proxy/myenv/bin/Activate.ps1
myenv/*
litellm/proxy/_experimental/out/404/index.html
litellm/proxy/_experimental/out/model_hub/index.html
litellm/proxy/_experimental/out/onboarding/index.html
litellm/tests/log.txt
litellm/tests/langfuse.log
litellm/tests/langfuse.log
@ -72,7 +71,32 @@ tests/local_testing/log.txt
.codegpt
litellm/proxy/_new_new_secret_config.yaml
litellm/proxy/custom_guardrail.py
litellm/proxy/_experimental/out/404.html
litellm/proxy/_experimental/out/404.html
litellm/proxy/_experimental/out/model_hub.html
.mypy_cache/*
.mypy_cache/*
litellm/proxy/application.log
tests/llm_translation/vertex_test_account.json
tests/llm_translation/test_vertex_key.json
litellm/proxy/migrations/0_init/migration.sql
litellm/proxy/db/migrations/0_init/migration.sql
litellm/proxy/db/migrations/*
litellm/proxy/migrations/*config.yaml
litellm/proxy/migrations/*
litellm/proxy/to_delete_loadtest_work/*
config.yaml
tests/litellm/litellm_core_utils/llm_cost_calc/log.txt
tests/test_custom_dir/*
test.py
litellm_config.yaml
.cursor
.vscode/launch.json
litellm/proxy/to_delete_loadtest_work/*
update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
scripts/test_vertex_ai_search.py
LAZY_LOADING_IMPROVEMENTS.md
STABILIZATION_TODO.md
**/test-results
**/playwright-report
**/*.storageState.json
**/coverage

33
.pre-commit-config.yaml
View file

@ -6,44 +6,35 @@ repos:
entry: pyright
language: system
types: [python]
files: ^litellm/
files: ^(litellm/|litellm_proxy_extras/|enterprise/)
- id: isort
name: isort
entry: isort
language: system
types: [python]
files: litellm/.*\.py
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
exclude: ^litellm/__init__.py$
- repo: https://github.com/psf/black
rev: 24.2.0
hooks:
- id: black
# - id: black
# name: black
# entry: poetry run black
# language: system
# types: [python]
# files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- repo: https://github.com/pycqa/flake8
rev: 7.0.0 # The version of flake8 to use
hooks:
- id: flake8
exclude: ^litellm/tests/|^litellm/proxy/tests/
exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/
additional_dependencies: [flake8-print]
files: litellm/.*\.py
# - id: flake8
# name: flake8 (router.py function length)
# files: ^litellm/router\.py$
# args: [--max-function-length=40]
# # additional_dependencies: [flake8-functions]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- repo: https://github.com/python-poetry/poetry
rev: 1.8.0
hooks:
- id: poetry-check
files: ^(pyproject.toml|litellm-proxy-extras/pyproject.toml)$
- repo: local
hooks:
- id: check-files-match
name: Check if files match
entry: python3 ci_cd/check_files_match.py
language: system
# - id: check-file-length
# name: Check file length
# entry: python check_file_length.py
# args: ["10000"] # set your desired maximum number of lines
# language: python
# files: litellm/.*\.py
# exclude: ^litellm/tests/
language: system

12
.trivyignore Normal file
View file

@ -0,0 +1,12 @@
# LiteLLM Trivy Ignore File
# CVEs listed here are temporarily allowlisted pending fixes
# Next.js vulnerabilities in UI dashboard (next@14.2.35)
# Allowlisted: 2026-01-31, 7-day fix timeline
# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+
# HIGH: DoS via request deserialization
GHSA-h25m-26qc-wcjf
# MEDIUM: Image Optimizer DoS
CVE-2025-59471

190
AGENTS.md Normal file
View file

@ -0,0 +1,190 @@
# INSTRUCTIONS FOR LITELLM
This document provides comprehensive instructions for AI agents working in the LiteLLM repository.
## OVERVIEW
LiteLLM is a unified interface for 100+ LLMs that:
- Translates inputs to provider-specific completion, embedding, and image generation endpoints
- Provides consistent OpenAI-format output across all providers
- Includes retry/fallback logic across multiple deployments (Router)
- Offers a proxy server (LLM Gateway) with budgets, rate limits, and authentication
- Supports advanced features like function calling, streaming, caching, and observability
## REPOSITORY STRUCTURE
### Core Components
- `litellm/` - Main library code
- `llms/` - Provider-specific implementations (OpenAI, Anthropic, Azure, etc.)
- `proxy/` - Proxy server implementation (LLM Gateway)
- `router_utils/` - Load balancing and fallback logic
- `types/` - Type definitions and schemas
- `integrations/` - Third-party integrations (observability, caching, etc.)
### Key Directories
- `tests/` - Comprehensive test suites
- `docs/my-website/` - Documentation website
- `ui/litellm-dashboard/` - Admin dashboard UI
- `enterprise/` - Enterprise-specific features
## DEVELOPMENT GUIDELINES
### MAKING CODE CHANGES
1. **Provider Implementations**: When adding/modifying LLM providers:
- Follow existing patterns in `litellm/llms/{provider}/`
- Implement proper transformation classes that inherit from `BaseConfig`
- Support both sync and async operations
- Handle streaming responses appropriately
- Include proper error handling with provider-specific exceptions
2. **Type Safety**:
- Use proper type hints throughout
- Update type definitions in `litellm/types/`
- Ensure compatibility with both Pydantic v1 and v2
3. **Testing**:
- Add tests in appropriate `tests/` subdirectories
- Include both unit tests and integration tests
- Test provider-specific functionality thoroughly
- Consider adding load tests for performance-critical changes
### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
1. **Tremor is DEPRECATED, do not use Tremor components in new features/changes**
- The only exception is the Tremor Table component and its required Tremor Table sub components.
2. **Use Common Components as much as possible**:
- These are usually defined in the `common_components` directory
- Use these components as much as possible and avoid building new components unless needed
3. **Testing**:
- The codebase uses **Vitest** and **React Testing Library**
- **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
- **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)
- **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled
- **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present
- **Test names must start with "should"**: All test names should follow the pattern `it("should ...")`
- **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed
- **Structure tests properly**:
- First test should verify the component renders successfully
- Subsequent tests should focus on functionality and user interactions
- Use `waitFor` for async operations that aren't already awaited
- **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation
### IMPORTANT PATTERNS
1. **Function/Tool Calling**:
- LiteLLM standardizes tool calling across providers
- OpenAI format is the standard, with transformations for other providers
- See `litellm/llms/anthropic/chat/transformation.py` for complex tool handling
2. **Streaming**:
- All providers should support streaming where possible
- Use consistent chunk formatting across providers
- Handle both sync and async streaming
3. **Error Handling**:
- Use provider-specific exception classes
- Maintain consistent error formats across providers
- Include proper retry logic and fallback mechanisms
4. **Configuration**:
- Support both environment variables and programmatic configuration
- Use `BaseConfig` classes for provider configurations
- Allow dynamic parameter passing
## PROXY SERVER (LLM GATEWAY)
The proxy server is a critical component that provides:
- Authentication and authorization
- Rate limiting and budget management
- Load balancing across multiple models/deployments
- Observability and logging
- Admin dashboard UI
- Enterprise features
Key files:
- `litellm/proxy/proxy_server.py` - Main server implementation
- `litellm/proxy/auth/` - Authentication logic
- `litellm/proxy/management_endpoints/` - Admin API endpoints
## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
LiteLLM supports MCP for agent workflows:
- MCP server integration for tool calling
- Transformation between OpenAI and MCP tool formats
- Support for external MCP servers (Zapier, Jira, Linear, etc.)
- See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/`
## RUNNING SCRIPTS
Use `poetry run python script.py` to run Python scripts in the project environment (for non-test files).
## GITHUB TEMPLATES
When opening issues or pull requests, follow these templates:
### Bug Reports (`.github/ISSUE_TEMPLATE/bug_report.yml`)
- Describe what happened vs. expected behavior
- Include relevant log output
- Specify LiteLLM version
- Indicate if you're part of an ML Ops team (helps with prioritization)
### Feature Requests (`.github/ISSUE_TEMPLATE/feature_request.yml`)
- Clearly describe the feature
- Explain motivation and use case with concrete examples
### Pull Requests (`.github/pull_request_template.md`)
- Add at least 1 test in `tests/litellm/`
- Ensure `make test-unit` passes
## TESTING CONSIDERATIONS
1. **Provider Tests**: Test against real provider APIs when possible
2. **Proxy Tests**: Include authentication, rate limiting, and routing tests
3. **Performance Tests**: Load testing for high-throughput scenarios
4. **Integration Tests**: End-to-end workflows including tool calling
## DOCUMENTATION
- Keep documentation in sync with code changes
- Update provider documentation when adding new providers
- Include code examples for new features
- Update changelog and release notes
## SECURITY CONSIDERATIONS
- Handle API keys securely
- Validate all inputs, especially for proxy endpoints
- Consider rate limiting and abuse prevention
- Follow security best practices for authentication
## ENTERPRISE FEATURES
- Some features are enterprise-only
- Check `enterprise/` directory for enterprise-specific code
- Maintain compatibility between open-source and enterprise versions
## COMMON PITFALLS TO AVOID
1. **Breaking Changes**: LiteLLM has many users - avoid breaking existing APIs
2. **Provider Specifics**: Each provider has unique quirks - handle them properly
3. **Rate Limits**: Respect provider rate limits in tests
4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
5. **Dependencies**: Keep dependencies minimal and well-justified
## HELPFUL RESOURCES
- Main documentation: https://docs.litellm.ai/
- Provider-specific docs in `docs/my-website/docs/providers/`
- Admin UI for testing proxy features
## WHEN IN DOUBT
- Follow existing patterns in the codebase
- Check similar provider implementations
- Ensure comprehensive test coverage
- Update documentation appropriately
- Consider backward compatibility impact

398
ARCHITECTURE.md Normal file
View file

@ -0,0 +1,398 @@
# LiteLLM Architecture - LiteLLM SDK + AI Gateway
This document helps contributors understand where to make changes in LiteLLM.
---
## How It Works
The LiteLLM AI Gateway (Proxy) uses the LiteLLM SDK internally for all LLM calls:
```
OpenAI SDK (client) ──▶ LiteLLM AI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
Anthropic SDK (client) ──▶ LiteLLMAI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
Any HTTP client ──▶ LiteLLMAI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
```
The **AI Gateway** adds authentication, rate limiting, budgets, and routing on top of the SDK.
The **SDK** handles the actual LLM provider calls, request/response transformations, and streaming.
---
## 1. AI Gateway (Proxy) Request Flow
The AI Gateway (`litellm/proxy/`) wraps the SDK with authentication, rate limiting, and management features.
```mermaid
sequenceDiagram
participant Client
participant ProxyServer as proxy/proxy_server.py
participant Auth as proxy/auth/user_api_key_auth.py
participant Redis as Redis Cache
participant Hooks as proxy/hooks/
participant Router as router.py
participant Main as main.py + utils.py
participant Handler as llms/custom_httpx/llm_http_handler.py
participant Transform as llms/{provider}/chat/transformation.py
participant Provider as LLM Provider API
participant CostCalc as cost_calculator.py
participant LoggingObj as litellm_logging.py
participant DBWriter as db/db_spend_update_writer.py
participant Postgres as PostgreSQL
%% Request Flow
Client->>ProxyServer: POST /v1/chat/completions
ProxyServer->>Auth: user_api_key_auth()
Auth->>Redis: Check API key cache
Redis-->>Auth: Key info + spend limits
ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter
Hooks->>Redis: Check/increment rate limit counters
ProxyServer->>Router: route_request()
Router->>Main: litellm.acompletion()
Main->>Handler: BaseLLMHTTPHandler.completion()
Handler->>Transform: ProviderConfig.transform_request()
Handler->>Provider: HTTP Request
Provider-->>Handler: Response
Handler->>Transform: ProviderConfig.transform_response()
Transform-->>Handler: ModelResponse
Handler-->>Main: ModelResponse
%% Cost Attribution (in utils.py wrapper)
Main->>LoggingObj: update_response_metadata()
LoggingObj->>CostCalc: _response_cost_calculator()
CostCalc->>CostCalc: completion_cost(tokens × price)
CostCalc-->>LoggingObj: response_cost
LoggingObj-->>Main: Set response._hidden_params["response_cost"]
Main-->>ProxyServer: ModelResponse (with cost in _hidden_params)
%% Response Headers + Async Logging
ProxyServer->>ProxyServer: Extract cost from hidden_params
ProxyServer->>LoggingObj: async_success_handler()
LoggingObj->>Hooks: async_log_success_event()
Hooks->>DBWriter: update_database(response_cost)
DBWriter->>Redis: Queue spend increment
DBWriter->>Postgres: Batch write spend logs (async)
ProxyServer-->>Client: ModelResponse + x-litellm-response-cost header
```
### Proxy Components
```mermaid
graph TD
subgraph "Incoming Request"
Client["POST /v1/chat/completions"]
end
subgraph "proxy/proxy_server.py"
Endpoint["chat_completion()"]
end
subgraph "proxy/auth/"
Auth["user_api_key_auth()"]
end
subgraph "proxy/"
PreCall["litellm_pre_call_utils.py"]
RouteRequest["route_llm_request.py"]
end
subgraph "litellm/"
Router["router.py"]
Main["main.py"]
end
subgraph "Infrastructure"
DualCache["DualCache<br/>(in-memory + Redis)"]
Postgres["PostgreSQL<br/>(keys, teams, spend logs)"]
end
Client --> Endpoint
Endpoint --> Auth
Auth --> DualCache
DualCache -.->|cache miss| Postgres
Auth --> PreCall
PreCall --> RouteRequest
RouteRequest --> Router
Router --> DualCache
Router --> Main
Main --> Client
```
**Key proxy files:**
- `proxy/proxy_server.py` - Main API endpoints
- `proxy/auth/` - Authentication (API keys, JWT, OAuth2)
- `proxy/hooks/` - Proxy-level callbacks
- `router.py` - Load balancing, fallbacks
- `router_strategy/` - Routing algorithms (`lowest_latency.py`, `simple_shuffle.py`, etc.)
**LLM-specific proxy endpoints:**
| Endpoint | Directory | Purpose |
|----------|-----------|---------|
| `/v1/messages` | `proxy/anthropic_endpoints/` | Anthropic Messages API |
| `/vertex-ai/*` | `proxy/vertex_ai_endpoints/` | Vertex AI passthrough |
| `/gemini/*` | `proxy/google_endpoints/` | Google AI Studio passthrough |
| `/v1/images/*` | `proxy/image_endpoints/` | Image generation |
| `/v1/batches` | `proxy/batches_endpoints/` | Batch processing |
| `/v1/files` | `proxy/openai_files_endpoints/` | File uploads |
| `/v1/fine_tuning` | `proxy/fine_tuning_endpoints/` | Fine-tuning jobs |
| `/v1/rerank` | `proxy/rerank_endpoints/` | Reranking |
| `/v1/responses` | `proxy/response_api_endpoints/` | OpenAI Responses API |
| `/v1/vector_stores` | `proxy/vector_store_endpoints/` | Vector stores |
| `/*` (passthrough) | `proxy/pass_through_endpoints/` | Direct provider passthrough |
**Proxy Hooks** (`proxy/hooks/__init__.py`):
| Hook | File | Purpose |
|------|------|---------|
| `max_budget_limiter` | `proxy/hooks/max_budget_limiter.py` | Enforce budget limits |
| `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user |
| `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation |
| `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation |
| `litellm_skills` | `proxy/hooks/skills_injection.py` | Skills injection |
To add a new proxy hook, implement `CustomLogger` and register in `PROXY_HOOKS`.
### Infrastructure Components
The AI Gateway uses external infrastructure for persistence and caching:
```mermaid
graph LR
subgraph "AI Gateway (proxy/)"
Proxy["proxy_server.py"]
Auth["auth/user_api_key_auth.py"]
DBWriter["db/db_spend_update_writer.py<br/>DBSpendUpdateWriter"]
InternalCache["utils.py<br/>InternalUsageCache"]
CostCallback["hooks/proxy_track_cost_callback.py<br/>_ProxyDBLogger"]
Scheduler["APScheduler<br/>ProxyStartupEvent"]
end
subgraph "SDK (litellm/)"
Router["router.py<br/>Router.cache (DualCache)"]
LLMCache["caching/caching_handler.py<br/>LLMCachingHandler"]
CacheClass["caching/caching.py<br/>Cache"]
end
subgraph "Redis (caching/redis_cache.py)"
RateLimit["Rate Limit Counters"]
SpendQueue["Spend Increment Queue"]
KeyCache["API Key Cache"]
TPM_RPM["TPM/RPM Tracking"]
Cooldowns["Deployment Cooldowns"]
LLMResponseCache["LLM Response Cache"]
end
subgraph "PostgreSQL (proxy/schema.prisma)"
Keys["LiteLLM_VerificationToken"]
Teams["LiteLLM_TeamTable"]
SpendLogs["LiteLLM_SpendLogs"]
Users["LiteLLM_UserTable"]
end
Auth --> InternalCache
InternalCache --> KeyCache
InternalCache -.->|cache miss| Keys
InternalCache --> RateLimit
Router --> TPM_RPM
Router --> Cooldowns
LLMCache --> CacheClass
CacheClass --> LLMResponseCache
CostCallback --> DBWriter
DBWriter --> SpendQueue
DBWriter --> SpendLogs
Scheduler --> SpendLogs
Scheduler --> Keys
```
| Component | Purpose | Key Files/Classes |
|-----------|---------|-------------------|
| **Redis** | Rate limiting, API key caching, TPM/RPM tracking, cooldowns, LLM response caching, spend queuing | `caching/redis_cache.py` (`RedisCache`), `caching/dual_cache.py` (`DualCache`) |
| **PostgreSQL** | API keys, teams, users, spend logs | `proxy/utils.py` (`PrismaClient`), `proxy/schema.prisma` |
| **InternalUsageCache** | Proxy-level cache for rate limits + API keys (in-memory + Redis) | `proxy/utils.py` (`InternalUsageCache`) |
| **Router.cache** | TPM/RPM tracking, deployment cooldowns, client caching (in-memory + Redis) | `router.py` (`Router.cache: DualCache`) |
| **LLMCachingHandler** | SDK-level LLM response/embedding caching | `caching/caching_handler.py` (`LLMCachingHandler`), `caching/caching.py` (`Cache`) |
| **DBSpendUpdateWriter** | Batches spend updates to reduce DB writes | `proxy/db/db_spend_update_writer.py` (`DBSpendUpdateWriter`) |
| **Cost Tracking** | Calculates and logs response costs | `proxy/hooks/proxy_track_cost_callback.py` (`_ProxyDBLogger`) |
**Background Jobs** (APScheduler, initialized in `proxy/proxy_server.py` → `ProxyStartupEvent.initialize_scheduled_background_jobs()`):
| Job | Interval | Purpose | Key Files |
|-----|----------|---------|-----------|
| `update_spend` | 60s | Batch write spend logs to PostgreSQL | `proxy/db/db_spend_update_writer.py` |
| `reset_budget` | 10-12min | Reset budgets for keys/users/teams | `proxy/management_helpers/budget_reset_job.py` |
| `add_deployment` | 10s | Sync new model deployments from DB | `proxy/proxy_server.py` (`ProxyConfig`) |
| `cleanup_old_spend_logs` | cron/interval | Delete old spend logs | `proxy/management_helpers/spend_log_cleanup.py` |
| `check_batch_cost` | 30min | Calculate costs for batch jobs | `proxy/management_helpers/check_batch_cost_job.py` |
| `check_responses_cost` | 30min | Calculate costs for responses API | `proxy/management_helpers/check_responses_cost_job.py` |
| `process_rotations` | 1hr | Auto-rotate API keys | `proxy/management_helpers/key_rotation_manager.py` |
| `_run_background_health_check` | continuous | Health check model deployments | `proxy/proxy_server.py` |
| `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
| `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
**Cost Attribution Flow:**
1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes
2. `update_response_metadata()` (`llm_response_utils/response_metadata.py`) is called
3. `logging_obj._response_cost_calculator()` (`litellm_logging.py`) calculates cost via `litellm.completion_cost()` (`cost_calculator.py`)
4. Cost is stored in `response._hidden_params["response_cost"]`
5. `proxy/common_request_processing.py` extracts cost from `hidden_params` and adds to response headers (`x-litellm-response-cost`)
6. `logging_obj.async_success_handler()` triggers callbacks including `_ProxyDBLogger.async_log_success_event()`
7. `DBSpendUpdateWriter.update_database()` queues spend increments to Redis
8. Background job `update_spend` flushes queued spend to PostgreSQL every 60s
---
## 2. SDK Request Flow
The SDK (`litellm/`) provides the core LLM calling functionality used by both direct SDK users and the AI Gateway.
```mermaid
graph TD
subgraph "SDK Entry Points"
Completion["litellm.completion()"]
Messages["litellm.messages()"]
end
subgraph "main.py"
Main["completion()<br/>acompletion()"]
end
subgraph "utils.py"
GetProvider["get_llm_provider()"]
end
subgraph "llms/custom_httpx/"
Handler["llm_http_handler.py<br/>BaseLLMHTTPHandler"]
HTTP["http_handler.py<br/>HTTPHandler / AsyncHTTPHandler"]
end
subgraph "llms/{provider}/chat/"
TransformReq["transform_request()"]
TransformResp["transform_response()"]
end
subgraph "litellm_core_utils/"
Streaming["streaming_handler.py"]
end
subgraph "integrations/ (async, off main thread)"
Callbacks["custom_logger.py<br/>Langfuse, Datadog, etc."]
end
Completion --> Main
Messages --> Main
Main --> GetProvider
GetProvider --> Handler
Handler --> TransformReq
TransformReq --> HTTP
HTTP --> Provider["LLM Provider API"]
Provider --> HTTP
HTTP --> TransformResp
TransformResp --> Streaming
Streaming --> Response["ModelResponse"]
Response -.->|async| Callbacks
```
**Key SDK files:**
- `main.py` - Entry points: `completion()`, `acompletion()`, `embedding()`
- `utils.py` - `get_llm_provider()` resolves model → provider
- `llms/custom_httpx/llm_http_handler.py` - Central HTTP orchestrator
- `llms/custom_httpx/http_handler.py` - Low-level HTTP client
- `llms/{provider}/chat/transformation.py` - Provider-specific transformations
- `litellm_core_utils/streaming_handler.py` - Streaming response handling
- `integrations/` - Async callbacks (Langfuse, Datadog, etc.)
---
## 3. Translation Layer
When a request comes in, it goes through a **translation layer** that converts between API formats.
Each translation is isolated in its own file, making it easy to test and modify independently.
### Where to find translations
| Incoming API | Provider | Translation File |
|--------------|----------|------------------|
| `/v1/chat/completions` | Anthropic | `llms/anthropic/chat/transformation.py` |
| `/v1/chat/completions` | Bedrock Converse | `llms/bedrock/chat/converse_transformation.py` |
| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` |
| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |
### Example: Debugging prompt caching
If `/v1/messages` → Bedrock Converse prompt caching isn't working but Bedrock Invoke works:
1. **Bedrock Converse translation**: `llms/bedrock/chat/converse_transformation.py`
2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py`
3. Compare how each handles `cache_control` in `transform_request()`
### How translations work
Each provider has a `Config` class that inherits from `BaseConfig` (`llms/base_llm/chat/transformation.py`):
```python
class ProviderConfig(BaseConfig):
def transform_request(self, model, messages, optional_params, litellm_params, headers):
# Convert OpenAI format → Provider format
return {"messages": transformed_messages, ...}
def transform_response(self, model, raw_response, model_response, logging_obj, ...):
# Convert Provider format → OpenAI format
return ModelResponse(choices=[...], usage=Usage(...))
```
The `BaseLLMHTTPHandler` (`llms/custom_httpx/llm_http_handler.py`) calls these methods - you never need to modify the handler itself.
---
## 4. Adding/Modifying Providers
### To add a new provider:
1. Create `llms/{provider}/chat/transformation.py`
2. Implement `Config` class with `transform_request()` and `transform_response()`
3. Add tests in `tests/llm_translation/test_{provider}.py`
### To add a feature (e.g., prompt caching):
1. Find the translation file from the table above
2. Modify `transform_request()` to handle the new parameter
3. Add unit tests that verify the transformation
### Testing checklist
When adding a feature, verify it works across all paths:
| Test | File Pattern |
|------|--------------|
| OpenAI passthrough | `tests/llm_translation/test_openai*.py` |
| Anthropic direct | `tests/llm_translation/test_anthropic*.py` |
| Bedrock Invoke | `tests/llm_translation/test_bedrock*.py` |
| Bedrock Converse | `tests/llm_translation/test_bedrock*converse*.py` |
| Vertex AI | `tests/llm_translation/test_vertex*.py` |
| Gemini | `tests/llm_translation/test_gemini*.py` |
### Unit testing translations
Translations are designed to be unit testable without making API calls:
```python
from litellm.llms.bedrock.chat.converse_transformation import BedrockConverseConfig
def test_prompt_caching_transform():
config = BedrockConverseConfig()
result = config.transform_request(
model="anthropic.claude-3-opus",
messages=[{"role": "user", "content": "test", "cache_control": {"type": "ephemeral"}}],
optional_params={},
litellm_params={},
headers={}
)
assert "cachePoint" in str(result) # Verify cache_control was translated
```

108
CLAUDE.md Normal file
View file

@ -0,0 +1,108 @@
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Development Commands
### Installation
- `make install-dev` - Install core development dependencies
- `make install-proxy-dev` - Install proxy development dependencies with full feature set
- `make install-test-deps` - Install all test dependencies
### Testing
- `make test` - Run all tests
- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
- `make test-integration` - Run integration tests (excludes unit tests)
- `pytest tests/` - Direct pytest execution
### Code Quality
- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
- `make format` - Apply Black code formatting
- `make lint-ruff` - Run Ruff linting only
- `make lint-mypy` - Run MyPy type checking only
### Single Test Files
- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
### Running Scripts
- `poetry run python script.py` - Run Python scripts (use for non-test files)
### GitHub Issue & PR Templates
When contributing to the project, use the appropriate templates:
**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
- Describe what happened vs. what you expected
- Include relevant log output
- Specify your LiteLLM version
**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
- Describe the feature clearly
- Explain the motivation and use case
**Pull Requests** (`.github/pull_request_template.md`):
- Add at least 1 test in `tests/litellm/`
- Ensure `make test-unit` passes
## Architecture Overview
LiteLLM is a unified interface for 100+ LLM providers with two main components:
### Core Library (`litellm/`)
- **Main entry point**: `litellm/main.py` - Contains core completion() function
- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
- **Type definitions**: `litellm/types/` - Pydantic models and type hints
- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
### Proxy Server (`litellm/proxy/`)
- **Main server**: `proxy_server.py` - FastAPI application
- **Authentication**: `auth/` - API key management, JWT, OAuth2
- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
- **Guardrails**: `guardrails/` - Safety and content filtering hooks
- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
## Key Patterns
### Provider Implementation
- Providers inherit from base classes in `litellm/llms/base.py`
- Each provider has transformation functions for input/output formatting
- Support both sync and async operations
- Handle streaming responses and function calling
### Error Handling
- Provider-specific exceptions mapped to OpenAI-compatible errors
- Fallback logic handled by Router system
- Comprehensive logging through `litellm/_logging.py`
### Configuration
- YAML config files for proxy server (see `proxy/example_config_yaml/`)
- Environment variables for API keys and settings
- Database schema managed via Prisma (`proxy/schema.prisma`)
## Development Notes
### Code Style
- Uses Black formatter, Ruff linter, MyPy type checker
- Pydantic v2 for data validation
- Async/await patterns throughout
- Type hints required for all public APIs
### Testing Strategy
- Unit tests in `tests/test_litellm/`
- Integration tests for each provider in `tests/llm_translation/`
- Proxy tests in `tests/proxy_unit_tests/`
- Load tests in `tests/load_tests/`
### Database Migrations
- Prisma handles schema migrations
- Migration files auto-generated with `prisma migrate dev`
- Always test migrations against both PostgreSQL and SQLite
### Enterprise Features
- Enterprise-specific code in `enterprise/` directory
- Optional features enabled via environment variables
- Separate licensing and authentication for enterprise features

276
CONTRIBUTING.md Normal file
View file

@ -0,0 +1,276 @@
# Contributing to LiteLLM
Thank you for your interest in contributing to LiteLLM! We welcome contributions of all kinds - from bug fixes and documentation improvements to new features and integrations.
## **Checklist before submitting a PR**
Here are the core requirements for any PR submitted to LiteLLM:
- [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
- [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
- [ ] **Ensure your PR passes all checks**:
- [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
- [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
## **Contributor License Agreement (CLA)**
Before contributing code to LiteLLM, you must sign our [Contributor License Agreement (CLA)](https://cla-assistant.io/BerriAI/litellm). This is a legal requirement for all contributions to be merged into the main repository.
**Important:** We strongly recommend reviewing and signing the CLA before starting work on your contribution to avoid any delays in the PR process.
## Quick Start
### 1. Setup Your Local Development Environment
```bash
# Fork the repository on GitHub (click the Fork button at https://github.com/BerriAI/litellm)
# Then clone your fork locally
git clone https://github.com/YOUR_USERNAME/litellm.git
cd litellm
# Create a new branch for your feature
git checkout -b your-feature-branch
# Install development dependencies
make install-dev
# Verify your setup works
make help
```
That's it! Your local development environment is ready.
### 2. Development Workflow
Here's the recommended workflow for making changes:
```bash
# Make your changes to the code
# ...
# Format your code (auto-fixes formatting issues)
make format
# Run all linting checks (matches CI exactly)
make lint
# Run unit tests to ensure nothing is broken
make test-unit
# Commit your changes
git add .
git commit -m "Your descriptive commit message"
# Push and create a PR
git push origin your-feature-branch
```
## Adding Testing
**Adding at least 1 test is a hard requirement for all PRs.**
### Where to Add Tests
Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm).
- This directory mirrors the structure of the `litellm/` directory
- **Only add mocked tests** - no real LLM API calls in this directory
- For integration tests with real APIs, use the appropriate test directories
### File Naming Convention
The `tests/test_litellm/` directory follows the same structure as `litellm/`:
- `litellm/proxy/caching_routes.py` → `tests/test_litellm/proxy/test_caching_routes.py`
- `litellm/utils.py` → `tests/test_litellm/test_utils.py`
### Example Test
```python
import pytest
from litellm import completion
def test_your_feature():
"""Test your feature with a descriptive docstring."""
# Arrange
messages = [{"role": "user", "content": "Hello"}]
# Act
# Use mocked responses, not real API calls
# Assert
assert expected_result == actual_result
```
## Running Tests and Checks
### Running Unit Tests
Run all unit tests (uses parallel execution for speed):
```bash
make test-unit
```
Run specific test files:
```bash
poetry run pytest tests/test_litellm/test_your_file.py -v
```
### Running Linting and Formatting Checks
Run all linting checks (matches CI exactly):
```bash
make lint
```
Individual linting commands:
```bash
make format-check # Check Black formatting
make lint-ruff # Run Ruff linting
make lint-mypy # Run MyPy type checking
make check-circular-imports # Check for circular imports
make check-import-safety # Check import safety
```
Apply formatting (auto-fixes issues):
```bash
make format
```
### CI Compatibility
To ensure your changes will pass CI, run the exact same checks locally:
```bash
# This runs the same checks as the GitHub workflows
make lint
make test-unit
```
For exact CI compatibility (pins OpenAI version like CI):
```bash
make install-dev-ci # Installs exact CI dependencies
```
## Available Make Commands
Run `make help` to see all available commands:
```bash
make help # Show all available commands
make install-dev # Install development dependencies
make install-proxy-dev # Install proxy development dependencies
make install-test-deps # Install test dependencies (for running tests)
make format # Apply Black code formatting
make format-check # Check Black formatting (matches CI)
make lint # Run all linting checks
make test-unit # Run unit tests
make test-integration # Run integration tests
make test-unit-helm # Run Helm unit tests
```
## Code Quality Standards
LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
Our automated quality checks include:
- **Black** for consistent code formatting
- **Ruff** for linting and code quality
- **MyPy** for static type checking
- **Circular import detection**
- **Import safety validation**
All checks must pass before your PR can be merged.
## Common Issues and Solutions
### 1. Linting Failures
If `make lint` fails:
1. **Formatting issues**: Run `make format` to auto-fix
2. **Ruff issues**: Check the output and fix manually
3. **MyPy issues**: Add proper type hints
4. **Circular imports**: Refactor import dependencies
5. **Import safety**: Fix any unprotected imports
### 2. Test Failures
If `make test-unit` fails:
1. Check if you broke existing functionality
2. Add tests for your new code
3. Ensure tests use mocks, not real API calls
4. Check test file naming conventions
### 3. Common Development Tips
- **Use type hints**: MyPy requires proper type annotations
- **Write descriptive commit messages**: Help reviewers understand your changes
- **Keep PRs focused**: One feature/fix per PR
- **Test edge cases**: Don't just test the happy path
- **Update documentation**: If you change APIs, update docs
## Building and Running Locally
### LiteLLM Proxy Server
To run the proxy server locally:
```bash
# Install proxy dependencies
make install-proxy-dev
# Start the proxy server
poetry run litellm --config your_config.yaml
```
### Docker Development
If you want to build the Docker image yourself:
```bash
# Build using the non-root Dockerfile
docker build -f docker/Dockerfile.non_root -t litellm_dev .
# Run with your config
docker run \
-v $(pwd)/proxy_config.yaml:/app/config.yaml \
-e LITELLM_MASTER_KEY="sk-1234" \
-p 4000:4000 \
litellm_dev \
--config /app/config.yaml --detailed_debug
```
## Submitting Your PR
1. **Push your branch**: `git push origin your-feature-branch`
2. **Create a PR**: Go to GitHub and create a pull request
3. **Fill out the PR template**: Provide clear description of changes
4. **Wait for review**: Maintainers will review and provide feedback
5. **Address feedback**: Make requested changes and push updates
6. **Merge**: Once approved, your PR will be merged!
## Getting Help
If you need help:
- 💬 [Join our Discord](https://discord.gg/wuPM9dRgDw)
- 💬 [Join our Slack](https://www.litellm.ai/support)
- 📧 Email us: ishaan@berri.ai / krrish@berri.ai
- 🐛 [Create an issue](https://github.com/BerriAI/litellm/issues/new)
## What to Contribute
Looking for ideas? Check out:
- 🐛 [Good first issues](https://github.com/BerriAI/litellm/labels/good%20first%20issue)
- 🚀 [Feature requests](https://github.com/BerriAI/litellm/labels/enhancement)
- 📚 Documentation improvements
- 🧪 Test coverage improvements
- 🔌 New LLM provider integrations
Thank you for contributing to LiteLLM! 🚀

48
Dockerfile
View file

@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
@ -12,18 +12,16 @@ WORKDIR /app
USER root
# Install build dependencies
RUN apk update && \
apk add --no-cache gcc python3-dev openssl openssl-dev
RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev
RUN pip install --upgrade pip && \
pip install build
RUN python -m pip install build
# Copy the current directory contents into the container at /app
COPY . .
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Build the package
RUN rm -rf dist/* && python -m build
@ -37,26 +35,20 @@ RUN pip install dist/*.whl
# install dependencies as wheels
RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
# install semantic-cache [Experimental]- we need this here and not in requirements.txt because redisvl pins to pydantic 1.0
RUN pip install redisvl==0.0.7 --no-deps
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT==2.9.0 --no-cache-dir
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE AS runtime
# Ensure runtime stage runs as root
USER root
# Install runtime dependencies
RUN apk update && \
apk add --no-cache openssl
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@latest
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -70,14 +62,26 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Generate prisma client
RUN prisma generate
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
# Remove test files and keys from dependencies
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
# Install semantic_router and aurelio-sdk using script
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Generate prisma client using the correct schema
RUN prisma generate --schema=./litellm/proxy/schema.prisma
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
RUN apk add --no-cache supervisor
COPY docker/supervisord.conf /etc/supervisord.conf
ENTRYPOINT ["docker/prod_entrypoint.sh"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
CMD ["--port", "4000"]

108
GEMINI.md Normal file
View file

@ -0,0 +1,108 @@
# GEMINI.md
This file provides guidance to Gemini when working with code in this repository.
## Development Commands
### Installation
- `make install-dev` - Install core development dependencies
- `make install-proxy-dev` - Install proxy development dependencies with full feature set
- `make install-test-deps` - Install all test dependencies
### Testing
- `make test` - Run all tests
- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
- `make test-integration` - Run integration tests (excludes unit tests)
- `pytest tests/` - Direct pytest execution
### Code Quality
- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
- `make format` - Apply Black code formatting
- `make lint-ruff` - Run Ruff linting only
- `make lint-mypy` - Run MyPy type checking only
### Single Test Files
- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
### Running Scripts
- `poetry run python script.py` - Run Python scripts (use for non-test files)
### GitHub Issue & PR Templates
When contributing to the project, use the appropriate templates:
**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
- Describe what happened vs. what you expected
- Include relevant log output
- Specify your LiteLLM version
**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
- Describe the feature clearly
- Explain the motivation and use case
**Pull Requests** (`.github/pull_request_template.md`):
- Add at least 1 test in `tests/litellm/`
- Ensure `make test-unit` passes
## Architecture Overview
LiteLLM is a unified interface for 100+ LLM providers with two main components:
### Core Library (`litellm/`)
- **Main entry point**: `litellm/main.py` - Contains core completion() function
- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
- **Type definitions**: `litellm/types/` - Pydantic models and type hints
- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
### Proxy Server (`litellm/proxy/`)
- **Main server**: `proxy_server.py` - FastAPI application
- **Authentication**: `auth/` - API key management, JWT, OAuth2
- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
- **Guardrails**: `guardrails/` - Safety and content filtering hooks
- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
## Key Patterns
### Provider Implementation
- Providers inherit from base classes in `litellm/llms/base.py`
- Each provider has transformation functions for input/output formatting
- Support both sync and async operations
- Handle streaming responses and function calling
### Error Handling
- Provider-specific exceptions mapped to OpenAI-compatible errors
- Fallback logic handled by Router system
- Comprehensive logging through `litellm/_logging.py`
### Configuration
- YAML config files for proxy server (see `proxy/example_config_yaml/`)
- Environment variables for API keys and settings
- Database schema managed via Prisma (`proxy/schema.prisma`)
## Development Notes
### Code Style
- Uses Black formatter, Ruff linter, MyPy type checker
- Pydantic v2 for data validation
- Async/await patterns throughout
- Type hints required for all public APIs
### Testing Strategy
- Unit tests in `tests/test_litellm/`
- Integration tests for each provider in `tests/llm_translation/`
- Proxy tests in `tests/proxy_unit_tests/`
- Load tests in `tests/load_tests/`
### Database Migrations
- Prisma handles schema migrations
- Migration files auto-generated with `prisma migrate dev`
- Always test migrations against both PostgreSQL and SQLite
### Enterprise Features
- Enterprise-specific code in `enterprise/` directory
- Optional features enabled via environment variables
- Separate licensing and authentication for enterprise features

149
Makefile Normal file
View file

@ -0,0 +1,149 @@
# LiteLLM Makefile
# Simple Makefile for running tests and basic development tasks
.PHONY: help test test-unit test-integration test-unit-helm \
info lint lint-dev format \
install-dev install-proxy-dev install-test-deps \
install-helm-unittest check-circular-imports check-import-safety
# Default target
help:
@echo "Available commands:"
@echo " make install-dev - Install development dependencies"
@echo " make install-proxy-dev - Install proxy development dependencies"
@echo " make install-dev-ci - Install dev dependencies (CI-compatible, pins OpenAI)"
@echo " make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)"
@echo " make install-test-deps - Install test dependencies"
@echo " make install-helm-unittest - Install helm unittest plugin"
@echo " make format - Apply Black code formatting"
@echo " make format-check - Check Black code formatting (matches CI)"
@echo " make lint - Run all linting (Ruff, MyPy, Black check, circular imports, import safety)"
@echo " make lint-ruff - Run Ruff linting only"
@echo " make lint-mypy - Run MyPy type checking only"
@echo " make lint-black - Check Black formatting (matches CI)"
@echo " make check-circular-imports - Check for circular imports"
@echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
@echo " make test-unit - Run unit tests (tests/test_litellm)"
@echo " make test-integration - Run integration tests"
@echo " make test-unit-helm - Run helm unit tests"
# Keep PIP simple for edge cases:
PIP := $(shell command -v pip > /dev/null 2>&1 && echo "pip" || echo "python3 -m pip")
# Show info
info:
@echo "PIP: $(PIP)"
# Installation targets
install-dev:
poetry install --with dev
install-proxy-dev:
poetry install --with dev,proxy-dev --extras proxy
# CI-compatible installations (matches GitHub workflows exactly)
install-dev-ci:
$(PIP) install openai==2.8.0
poetry install --with dev
$(PIP) install openai==2.8.0
install-proxy-dev-ci:
poetry install --with dev,proxy-dev --extras proxy
$(PIP) install openai==2.8.0
install-test-deps: install-proxy-dev
poetry run $(PIP) install "pytest-retry==1.6.3"
poetry run $(PIP) install pytest-xdist
poetry run $(PIP) install openapi-core
cd enterprise && poetry run $(PIP) install -e . && cd ..
install-helm-unittest:
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
# Formatting
format: install-dev
cd litellm && poetry run black . && cd ..
format-check: install-dev
cd litellm && poetry run black --check . && cd ..
# Linting targets
lint-ruff: install-dev
cd litellm && poetry run ruff check . && cd ..
# faster linter for developing ...
# inspiration from:
# https://github.com/astral-sh/ruff/discussions/10977
# https://github.com/astral-sh/ruff/discussions/4049
lint-format-changed: install-dev
@git diff origin/main --unified=0 --no-color -- '*.py' | \
perl -ne '\
if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
print "$$file:$$start:1-$$end:999\n"; \
}' | \
while read range; do \
file="$${range%%:*}"; \
lines="$${range#*:}"; \
echo "Formatting $$file (lines $$lines)"; \
poetry run ruff format --range "$$lines" "$$file"; \
done
lint-ruff-dev: install-dev
@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
cd litellm && \
(poetry run ruff check . --output-format=pylint || true) > "$$tmpfile" && \
poetry run diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
cd .. ; \
rm -f "$$tmpfile"
lint-ruff-FULL-dev: install-dev
@files=$$(git diff --name-only origin/main -- '*.py'); \
if [ -n "$$files" ]; then echo "$$files" | xargs poetry run ruff check; \
else echo "No changed .py files to check."; fi
lint-mypy: install-dev
poetry run $(PIP) install types-requests types-setuptools types-redis types-PyYAML
cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
lint-black: format-check
check-circular-imports: install-dev
cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
check-import-safety: install-dev
@poetry run python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Combined linting (matches test-linting.yml workflow)
lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
# Testing targets
test:
poetry run pytest tests/
test-unit: install-test-deps
poetry run pytest tests/test_litellm -x -vv -n 4
test-integration:
poetry run pytest tests/ -k "not test_litellm"
test-unit-helm: install-helm-unittest
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
# LLM Translation testing targets
test-llm-translation: install-test-deps
@echo "Running LLM translation tests..."
@python .github/workflows/run_llm_translation_tests.py
test-llm-translation-single: install-test-deps
@echo "Running single LLM translation test file..."
@if [ -z "$(FILE)" ]; then echo "Usage: make test-llm-translation-single FILE=test_filename.py"; exit 1; fi
@mkdir -p test-results
poetry run pytest tests/llm_translation/$(FILE) \
--junitxml=test-results/junit.xml \
-v --tb=short --maxfail=100 --timeout=300

694
README.md
View file

@ -2,23 +2,20 @@
🚅 LiteLLM
</h1>
<p align="center">
<p align="center">Call 100+ LLMs in OpenAI format. [Bedrock, Azure, OpenAI, VertexAI, Anthropic, Groq, etc.]
</p>
<p align="center">
<a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
<a href="https://railway.app/template/HLP0Ub?referralCode=jch2ME">
<img src="https://railway.app/button.svg" alt="Deploy on Railway">
</a>
</p>
<p align="center">Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]
<br>
</p>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (LLM Gateway)</a> | <a href="https://docs.litellm.ai/docs/hosted" target="_blank"> Hosted Proxy (Preview)</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
<h4 align="center">
<a href="https://pypi.org/project/litellm/" target="_blank">
<img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
</a>
<a href="https://dl.circleci.com/status-badge/redirect/gh/BerriAI/litellm/tree/main" target="_blank">
<img src="https://dl.circleci.com/status-badge/img/gh/BerriAI/litellm/tree/main.svg?style=svg" alt="CircleCI">
</a>
<a href="https://www.ycombinator.com/companies/berriai">
<img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23">
</a>
@ -28,31 +25,22 @@
<a href="https://discord.gg/wuPM9dRgDw">
<img src="https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square" alt="Discord">
</a>
<a href="https://www.litellm.ai/support">
<img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
</a>
</h4>
LiteLLM manages:
<img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) <br>
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
## Use LiteLLM for
🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published.
<details open>
<summary><b>LLMs</b> - Call 100+ LLMs (Python SDK + AI Gateway)</summary>
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
[**All Supported Endpoints**](https://docs.litellm.ai/docs/supported_endpoints) - `/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, `/rerank`, `/a2a`, `/messages` and more.
# Usage ([**Docs**](https://docs.litellm.ai/docs/))
> [!IMPORTANT]
> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
> LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
</a>
### Python SDK
```shell
pip install litellm
@ -62,349 +50,358 @@ pip install litellm
from litellm import completion
import os
## set ENV variables
os.environ["OPENAI_API_KEY"] = "your-openai-key"
os.environ["ANTHROPIC_API_KEY"] = "your-cohere-key"
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
messages = [{ "content": "Hello, how are you?","role": "user"}]
# OpenAI
response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hello!"}])
# openai call
response = completion(model="openai/gpt-4o", messages=messages)
# anthropic call
response = completion(model="anthropic/claude-3-sonnet-20240229", messages=messages)
print(response)
# Anthropic
response = completion(model="anthropic/claude-sonnet-4-20250514", messages=[{"role": "user", "content": "Hello!"}])
```
### Response (OpenAI Format)
### AI Gateway (Proxy Server)
```json
{
"id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885",
"created": 1734366691,
"model": "claude-3-sonnet-20240229",
"object": "chat.completion",
"system_fingerprint": null,
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Hello! As an AI language model, I don't have feelings, but I'm operating properly and ready to assist you with any questions or tasks you may have. How can I help you today?",
"role": "assistant",
"tool_calls": null,
"function_call": null
}
}
],
"usage": {
"completion_tokens": 43,
"prompt_tokens": 13,
"total_tokens": 56,
"completion_tokens_details": null,
"prompt_tokens_details": {
"audio_tokens": null,
"cached_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0
}
}
```
Call any model supported by a provider, with `model=<provider_name>/<model_name>`. There might be provider-specific details here, so refer to [provider docs for more information](https://docs.litellm.ai/docs/providers)
## Async ([Docs](https://docs.litellm.ai/docs/completion/stream#async-completion))
```python
from litellm import acompletion
import asyncio
async def test_get_response():
user_message = "Hello, how are you?"
messages = [{"content": user_message, "role": "user"}]
response = await acompletion(model="openai/gpt-4o", messages=messages)
return response
response = asyncio.run(test_get_response())
print(response)
```
## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
```python
from litellm import completion
response = completion(model="openai/gpt-4o", messages=messages, stream=True)
for part in response:
print(part.choices[0].delta.content or "")
# claude 2
response = completion('anthropic/claude-3-sonnet-20240229', messages, stream=True)
for part in response:
print(part)
```
### Response chunk (OpenAI Format)
```json
{
"id": "chatcmpl-2be06597-eb60-4c70-9ec5-8cd2ab1b4697",
"created": 1734366925,
"model": "claude-3-sonnet-20240229",
"object": "chat.completion.chunk",
"system_fingerprint": null,
"choices": [
{
"finish_reason": null,
"index": 0,
"delta": {
"content": "Hello",
"role": "assistant",
"function_call": null,
"tool_calls": null,
"audio": null
},
"logprobs": null
}
]
}
```
## Logging Observability ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack
```python
from litellm import completion
## set env variables for logging tools (when using MLflow, no API key set up is required)
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
os.environ["HELICONE_API_KEY"] = "your-helicone-auth-key"
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
os.environ["LANGFUSE_SECRET_KEY"] = ""
os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
os.environ["OPENAI_API_KEY"]
# set callbacks
litellm.success_callback = ["lunary", "mlflow", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
#openai call
response = completion(model="anthropic/claude-3-sonnet-20240229", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
```
# LiteLLM Proxy Server (LLM Gateway) - ([Docs](https://docs.litellm.ai/docs/simple_proxy))
Track spend + Load Balance across multiple projects
[Hosted Proxy (Preview)](https://docs.litellm.ai/docs/hosted)
The proxy provides:
1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth)
2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class)
3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend)
4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits)
## 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/)
## Quick Start Proxy - CLI
[**Getting Started - E2E Tutorial**](https://docs.litellm.ai/docs/proxy/docker_quick_start) - Setup virtual keys, make your first request
```shell
pip install 'litellm[proxy]'
litellm --model gpt-4o
```
### Step 1: Start litellm proxy
```shell
$ litellm --model huggingface/bigcode/starcoder
#INFO: Proxy running on http://0.0.0.0:4000
```
### Step 2: Make ChatCompletions Request to Proxy
> [!IMPORTANT]
> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
```python
import openai # openai v1.0.0+
client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
])
import openai
print(response)
client = openai.OpenAI(api_key="anything", base_url="http://0.0.0.0:4000")
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello!"}]
)
```
## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys))
[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
Connect the proxy with a Postgres DB to create proxy keys
</details>
<details>
<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
[**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
### Python SDK - A2A Protocol
```python
from litellm.a2a_protocol import A2AClient
from a2a.types import SendMessageRequest, MessageSendParams
from uuid import uuid4
client = A2AClient(base_url="http://localhost:10001")
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello!"}],
"messageId": uuid4().hex,
}
)
)
response = await client.send_message(request)
```
### AI Gateway (Proxy Server)
**Step 1.** [Add your Agent to the AI Gateway](https://docs.litellm.ai/docs/a2a#adding-your-agent)
**Step 2.** Call Agent via A2A SDK
```python
from a2a.client import A2ACardResolver, A2AClient
from a2a.types import MessageSendParams, SendMessageRequest
from uuid import uuid4
import httpx
base_url = "http://localhost:4000/a2a/my-agent" # LiteLLM proxy + agent name
headers = {"Authorization": "Bearer sk-1234"} # LiteLLM Virtual Key
async with httpx.AsyncClient(headers=headers) as httpx_client:
resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
agent_card = await resolver.get_agent_card()
client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello!"}],
"messageId": uuid4().hex,
}
)
)
response = await client.send_message(request)
```
[**Docs: A2A Agent Gateway**](https://docs.litellm.ai/docs/a2a)
</details>
<details>
<summary><b>MCP Tools</b> - Connect MCP servers to any LLM (Python SDK + AI Gateway)</summary>
### Python SDK - MCP Bridge
```python
from mcp import ClientSession, StdioServerParameters
from mcp.client.stdio import stdio_client
from litellm import experimental_mcp_client
import litellm
server_params = StdioServerParameters(command="python", args=["mcp_server.py"])
async with stdio_client(server_params) as (read, write):
async with ClientSession(read, write) as session:
await session.initialize()
# Load MCP tools in OpenAI format
tools = await experimental_mcp_client.load_mcp_tools(session=session, format="openai")
# Use with any LiteLLM model
response = await litellm.acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": "What's 3 + 5?"}],
tools=tools
)
```
### AI Gateway - MCP Gateway
**Step 1.** [Add your MCP Server to the AI Gateway](https://docs.litellm.ai/docs/mcp#adding-your-mcp)
**Step 2.** Call MCP tools via `/chat/completions`
```bash
# Get the code
git clone https://github.com/BerriAI/litellm
# Go to folder
cd litellm
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# Add the litellm salt key - you cannot change this after adding a model
# It is used to encrypt / decrypt your LLM API Key credentials
# We recommend - https://1password.com/password-generator/
# password generator to get a random hash for litellm salt key
echo 'LITELLM_SALT_KEY="sk-1234"' > .env
source .env
# Start
docker-compose up
curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Summarize the latest open PR"}],
"tools": [{
"type": "mcp",
"server_url": "litellm_proxy/mcp/github",
"server_label": "github_mcp",
"require_approval": "never"
}]
}'
```
### Use with Cursor IDE
UI on `/ui` on your proxy server
![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033)
Set budgets and rate limits across multiple projects
`POST /key/generate`
### Request
```shell
curl 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data-raw '{"models": ["gpt-3.5-turbo", "gpt-4", "claude-2"], "duration": "20m","metadata": {"user": "ishaan@berri.ai", "team": "core-infra"}}'
```
### Expected Response
```shell
```json
{
"key": "sk-kdEXbIqZRwEeEiHwdg7sFA", # Bearer token
"expires": "2023-11-19T01:38:25.838000+00:00" # datetime object
"mcpServers": {
"LiteLLM": {
"url": "http://localhost:4000/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
}
}
}
```
## Supported Providers ([Docs](https://docs.litellm.ai/docs/providers))
[**Docs: MCP Gateway**](https://docs.litellm.ai/docs/mcp)
| Provider | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) | [Async Image Generation](https://docs.litellm.ai/docs/image_generation) |
|-------------------------------------------------------------------------------------|---------------------------------------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------|-------------------------------------------------------------------------------|-------------------------------------------------------------------------|
| [openai](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [azure](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [AI/ML API](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [aws - bedrock](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [google - vertex_ai](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [google - palm](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | ✅ | | |
| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | | |
| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ | | |
| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ | | |
| [empower](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | ✅ |
| [huggingface](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [replicate](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | ✅ | | |
| [together_ai](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | ✅ | | |
| [openrouter](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | ✅ | | |
| [ai21](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | ✅ | | |
| [baseten](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | ✅ | | |
| [vllm](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | ✅ | | |
| [nlp_cloud](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | ✅ | | |
| [aleph alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | ✅ | | |
| [petals](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | ✅ | | |
| [ollama](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ | | |
| [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ | | |
| [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ | | |
| [Deepseek](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | ✅ | | |
| [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ | | |
| [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ | |
| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | |
| [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | ✅ | | |
| [Galadriel](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | ✅ | | |
</details>
---
## How to use LiteLLM
You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs:
<table style={{width: '100%', tableLayout: 'fixed'}}>
<thead>
<tr>
<th style={{width: '14%'}}></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
</tr>
</thead>
<tbody>
<tr>
<td style={{width: '14%'}}><strong>Use Case</strong></td>
<td style={{width: '43%'}}>Central service (LLM Gateway) to access multiple LLMs</td>
<td style={{width: '43%'}}>Use LiteLLM directly in your Python code</td>
</tr>
<tr>
<td style={{width: '14%'}}><strong>Who Uses It?</strong></td>
<td style={{width: '43%'}}>Gen AI Enablement / ML Platform Teams</td>
<td style={{width: '43%'}}>Developers building LLM projects</td>
</tr>
<tr>
<td style={{width: '14%'}}><strong>Key Features</strong></td>
<td style={{width: '43%'}}>Centralized API gateway with authentication and authorization, multi-tenant cost tracking and spend management per project/user, per-project customization (logging, guardrails, caching), virtual keys for secure access control, admin dashboard UI for monitoring and management</td>
<td style={{width: '43%'}}>Direct Python library integration in your codebase, Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - <a href="https://docs.litellm.ai/docs/routing">Router</a>, application-level load balancing and cost tracking, exception handling with OpenAI-compatible errors, observability callbacks (Lunary, MLflow, Langfuse, etc.)</td>
</tr>
</tbody>
</table>
LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://docs.litellm.ai/docs/simple_proxy) <br>
[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
**Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
## OSS Adopters
<table>
<tr>
<td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
<td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
<td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
<td><h2>Netflix</h2></td>
<td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
</tr>
</table>
## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
|-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ | | | | | | | | | |
| [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [Aleph Alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | | | | | | | |
| [Amazon Nova](https://docs.litellm.ai/docs/providers/amazon_nova) | ✅ | ✅ | ✅ | | | | | | | |
| [Anthropic (`anthropic`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
| [Anthropic Text (`anthropic_text`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
| [Anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | | | | | | | |
| [AssemblyAI (`assemblyai`)](https://docs.litellm.ai/docs/pass_through/assembly_ai) | ✅ | ✅ | ✅ | | | ✅ | | | | |
| [Auto Router (`auto_router`)](https://docs.litellm.ai/docs/proxy/auto_routing) | ✅ | ✅ | ✅ | | | | | | | |
| [AWS - Bedrock (`bedrock`)](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [AWS - Sagemaker (`sagemaker`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Azure (`azure`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [Azure AI (`azure_ai`)](https://docs.litellm.ai/docs/providers/azure_ai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [Azure Text (`azure_text`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Baseten (`baseten`)](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | | | | | | | |
| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | |
| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | |
| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | |
| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | |
| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Cohere (`cohere`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [Cohere Chat (`cohere_chat`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | | | | | | | |
| [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | | | | | | | |
| [Databricks (`databricks`)](https://docs.litellm.ai/docs/providers/databricks) | ✅ | ✅ | ✅ | | | | | | | |
| [DataRobot (`datarobot`)](https://docs.litellm.ai/docs/providers/datarobot) | ✅ | ✅ | ✅ | | | | | | | |
| [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ | | | ✅ | | | | |
| [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | | | | | | | |
| [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | | | | | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [FriendliAI (`friendliai`)](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | | | | | | | |
| [Galadriel (`galadriel`)](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | | | | | | | |
| [GitHub Copilot (`github_copilot`)](https://docs.litellm.ai/docs/providers/github_copilot) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [GitHub Models (`github`)](https://docs.litellm.ai/docs/providers/github) | ✅ | ✅ | ✅ | | | | | | | |
| [Google - PaLM](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | | | | | | | |
| [Google - Vertex AI (`vertex_ai`)](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [Google AI Studio - Gemini (`gemini`)](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | | | | | | | |
| [GradientAI (`gradient_ai`)](https://docs.litellm.ai/docs/providers/gradient_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Groq AI (`groq`)](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | | | | | | | |
| [Heroku (`heroku`)](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | ✅ | | | | | | | |
| [Hosted VLLM (`hosted_vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |
| [Huggingface (`huggingface`)](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [Hyperbolic (`hyperbolic`)](https://docs.litellm.ai/docs/providers/hyperbolic) | ✅ | ✅ | ✅ | | | | | | | |
| [IBM - Watsonx.ai (`watsonx`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Infinity (`infinity`)](https://docs.litellm.ai/docs/providers/infinity) | | | | ✅ | | | | | | |
| [Jina AI (`jina_ai`)](https://docs.litellm.ai/docs/providers/jina_ai) | | | | ✅ | | | | | | |
| [Lambda AI (`lambda_ai`)](https://docs.litellm.ai/docs/providers/lambda_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Lemonade (`lemonade`)](https://docs.litellm.ai/docs/providers/lemonade) | ✅ | ✅ | ✅ | | | | | | | |
| [LiteLLM Proxy (`litellm_proxy`)](https://docs.litellm.ai/docs/providers/litellm_proxy) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [Llamafile (`llamafile`)](https://docs.litellm.ai/docs/providers/llamafile) | ✅ | ✅ | ✅ | | | | | | | |
| [LM Studio (`lm_studio`)](https://docs.litellm.ai/docs/providers/lm_studio) | ✅ | ✅ | ✅ | | | | | | | |
| [Maritalk (`maritalk`)](https://docs.litellm.ai/docs/providers/maritalk) | ✅ | ✅ | ✅ | | | | | | | |
| [Meta - Llama API (`meta_llama`)](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ | | | | | | | |
| [Mistral AI API (`mistral`)](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | |
| [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | |
| [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | | | | | | | |
| [Nscale (`nscale`)](https://docs.litellm.ai/docs/providers/nscale) | ✅ | ✅ | ✅ | | | | | | | |
| [Nvidia NIM (`nvidia_nim`)](https://docs.litellm.ai/docs/providers/nvidia_nim) | ✅ | ✅ | ✅ | | | | | | | |
| [OCI (`oci`)](https://docs.litellm.ai/docs/providers/oci) | ✅ | ✅ | ✅ | | | | | | | |
| [Ollama (`ollama`)](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Ollama Chat (`ollama_chat`)](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | | | | | | | |
| [Oobabooga (`oobabooga`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [OpenAI (`openai`)](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) | | | | ✅ | | | | | | |
| [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | | | | | | | |
| [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | | | | | | | |
| [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | | | | | | | |
| [Predibase (`predibase`)](https://docs.litellm.ai/docs/providers/predibase) | ✅ | ✅ | ✅ | | | | | | | |
| [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) | | | | | ✅ | | | | | |
| [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | | | | | | | |
| [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | |
| [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | |
| [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | |
| [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | | | | | | | |
| [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ | | | | | | | |
| [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ | | | | | | | |
| [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ | | | | | | | |
| [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
| [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |
| [Volcengine (`volcengine`)](https://docs.litellm.ai/docs/providers/volcano) | ✅ | ✅ | ✅ | | | | | | | |
| [Voyage AI (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | |
| [WandB Inference (`wandb`)](https://docs.litellm.ai/docs/providers/wandb_inference) | ✅ | ✅ | ✅ | | | | | | | |
| [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | |
| [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | |
| [Xinference (`xinference`)](https://docs.litellm.ai/docs/providers/xinference) | | | | ✅ | | | | | | |
[**Read the Docs**](https://docs.litellm.ai/docs/)
## Contributing
## Run in Developer mode
### Services
1. Setup .env file in root
2. Run dependant services `docker-compose up db prometheus`
To contribute: Clone the repo locally -> Make a change -> Submit a PR with the change.
### Backend
1. (In root) create virtual environment `python -m venv .venv`
2. Activate virtual environment `source .venv/bin/activate`
3. Install dependencies `pip install -e ".[all]"`
4. `pip install prisma`
5. `prisma generate`
6. Start proxy backend `python litellm/proxy/proxy_cli.py`
Here's how to modify the repo locally:
Step 1: Clone the repo
```
git clone https://github.com/BerriAI/litellm.git
```
Step 2: Navigate into the project, and install dependencies:
```
cd litellm
poetry install -E extra_proxy -E proxy
```
Step 3: Test your change:
```
cd tests # pwd: Documents/litellm/litellm/tests
poetry run flake8
poetry run pytest .
```
Step 4: Submit a PR with your changes! 🚀
- push your fork to your GitHub repo
- submit a PR from there
### Building LiteLLM Docker Image
Follow these instructions if you want to build / run the LiteLLM Docker Image yourself.
Step 1: Clone the repo
```
git clone https://github.com/BerriAI/litellm.git
```
Step 2: Build the Docker Image
Build using Dockerfile.non_root
```
docker build -f docker/Dockerfile.non_root -t litellm_test_image .
```
Step 3: Run the Docker Image
Make sure config.yaml is present in the root directory. This is your litellm proxy config file.
```
docker run \
-v $(pwd)/proxy_config.yaml:/app/config.yaml \
-e DATABASE_URL="postgresql://xxxxxxxx" \
-e LITELLM_MASTER_KEY="sk-1234" \
-p 4000:4000 \
litellm_test_image \
--config /app/config.yaml --detailed_debug
```
### Frontend
1. Navigate to `ui/litellm-dashboard`
2. Install dependencies `npm install`
3. Run `npm run dev` to start the dashboard
# Enterprise
For companies that need better security, user management and professional support
[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
This covers:
This covers:
- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
- ✅ **Feature Prioritization**
- ✅ **Custom Integrations**
@ -412,24 +409,46 @@ This covers:
- ✅ **Custom SLAs**
- ✅ **Secure access with Single Sign-On**
# Code Quality / Linting
# Contributing
We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features, or improving documentation, we appreciate your help.
## Quick Start for Contributors
This requires poetry to be installed.
```bash
git clone https://github.com/BerriAI/litellm.git
cd litellm
make install-dev # Install development dependencies
make format # Format your code
make lint # Run all linting checks
make test-unit # Run unit tests
make format-check # Check formatting only
```
For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
## Code Quality / Linting
LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
We run:
- Ruff for [formatting and linting checks](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L320)
- Mypy + Pyright for typing [1](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L90), [2](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.pre-commit-config.yaml#L4)
- Black for [formatting](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L79)
- isort for [import sorting](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.pre-commit-config.yaml#L10)
Our automated checks include:
- **Black** for code formatting
- **Ruff** for linting and code quality
- **MyPy** for type checking
- **Circular import detection**
- **Import safety checks**
If you have suggestions on how to improve the code quality feel free to open an issue or a PR.
All these checks must pass before your PR can be merged.
# Support / talk with founders
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- [Community Slack 💭](https://www.litellm.ai/support)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
@ -451,3 +470,4 @@ If you have suggestions on how to improve the code quality feel free to open an
<a href="https://github.com/BerriAI/litellm/graphs/contributors">
<img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
</a>

3
ci_cd/.grype.yaml Normal file
View file

@ -0,0 +1,3 @@
ignore:
- vulnerability: CVE-2026-22184
reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists

40
ci_cd/TEST_KEY_PATTERNS.md Normal file
View file

@ -0,0 +1,40 @@
# Test Key Patterns Standard
Standard patterns for test/mock keys and credentials in the LiteLLM codebase to avoid triggering secret detection.
## How GitGuardian Works
GitGuardian uses **machine learning and entropy analysis**, not just pattern matching:
- **Low entropy** values (like `sk-1234`, `postgres`) are automatically ignored
- **High entropy** values (realistic-looking secrets) trigger detection
- **Context-aware** detection understands code syntax like `os.environ["KEY"]`
## Recommended Test Key Patterns
### Option 1: Low Entropy Values (Simplest)
These won't trigger GitGuardian's ML detector:
```python
api_key = "sk-1234"
api_key = "sk-12345"
database_password = "postgres"
token = "test123"
```
### Option 2: High Entropy with Test Prefixes
If you need realistic-looking test keys with high entropy, use these prefixes:
```python
api_key = "sk-test-abc123def456ghi789..." # OpenAI-style test key
api_key = "sk-mock-1234567890abcdef1234..." # Mock key
api_key = "sk-fake-xyz789uvw456rst123..." # Fake key
token = "test-api-key-with-high-entropy"
```
## Configured Ignore Patterns
These patterns are in `.gitguardian.yaml` for high-entropy test keys:
- `sk-test-*` - OpenAI-style test keys
- `sk-mock-*` - Mock API keys
- `sk-fake-*` - Fake API keys
- `test-api-key` - Generic test tokens

60
ci_cd/baseline_db.py Normal file
View file

@ -0,0 +1,60 @@
import subprocess
from pathlib import Path
from datetime import datetime
def create_baseline():
"""Create baseline migration in deploy/migrations"""
try:
# Get paths
root_dir = Path(__file__).parent.parent
deploy_dir = root_dir / "deploy"
migrations_dir = deploy_dir / "migrations"
schema_path = root_dir / "schema.prisma"
# Create migrations directory
migrations_dir.mkdir(parents=True, exist_ok=True)
# Create migration_lock.toml if it doesn't exist
lock_file = migrations_dir / "migration_lock.toml"
if not lock_file.exists():
lock_file.write_text('provider = "postgresql"\n')
# Create timestamp-based migration directory
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
migration_dir = migrations_dir / f"{timestamp}_baseline"
migration_dir.mkdir(parents=True, exist_ok=True)
# Generate migration SQL
result = subprocess.run(
[
"prisma",
"migrate",
"diff",
"--from-empty",
"--to-schema-datamodel",
str(schema_path),
"--script",
],
capture_output=True,
text=True,
check=True,
)
# Write the SQL to migration.sql
migration_file = migration_dir / "migration.sql"
migration_file.write_text(result.stdout)
print(f"Created baseline migration in {migration_dir}")
return True
except subprocess.CalledProcessError as e:
print(f"Error running prisma command: {e.stderr}")
return False
except Exception as e:
print(f"Error creating baseline migration: {str(e)}")
return False
if __name__ == "__main__":
create_baseline()

19
ci_cd/publish-proxy-extras.sh Normal file
View file

@ -0,0 +1,19 @@
#!/bin/bash
# Exit on error
set -e
echo "🚀 Building and publishing litellm-proxy-extras"
# Navigate to litellm-proxy-extras directory
cd "$(dirname "$0")/../litellm-proxy-extras"
# Build the package
echo "📦 Building package..."
poetry build
# Publish to PyPI
echo "🌎 Publishing to PyPI..."
poetry publish
echo "✅ Done! Package published successfully"

95
ci_cd/run_migration.py Normal file
View file

@ -0,0 +1,95 @@
import os
import subprocess
from pathlib import Path
from datetime import datetime
import testing.postgresql
import shutil
def create_migration(migration_name: str = None):
"""
Create a new migration SQL file in the migrations directory by comparing
current database state with schema
Args:
migration_name (str): Name for the migration
"""
try:
# Get paths
root_dir = Path(__file__).parent.parent
migrations_dir = root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
schema_path = root_dir / "schema.prisma"
# Create temporary PostgreSQL database
with testing.postgresql.Postgresql() as postgresql:
db_url = postgresql.url()
# Create temporary migrations directory next to schema.prisma
temp_migrations_dir = schema_path.parent / "migrations"
try:
# Copy existing migrations to temp directory
if temp_migrations_dir.exists():
shutil.rmtree(temp_migrations_dir)
shutil.copytree(migrations_dir, temp_migrations_dir)
# Apply existing migrations to temp database
os.environ["DATABASE_URL"] = db_url
subprocess.run(
["prisma", "migrate", "deploy", "--schema", str(schema_path)],
check=True,
)
# Generate diff between current database and schema
result = subprocess.run(
[
"prisma",
"migrate",
"diff",
"--from-url",
db_url,
"--to-schema-datamodel",
str(schema_path),
"--script",
],
capture_output=True,
text=True,
check=True,
)
if result.stdout.strip():
# Generate timestamp and create migration directory
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
migration_name = migration_name or "unnamed_migration"
migration_dir = migrations_dir / f"{timestamp}_{migration_name}"
migration_dir.mkdir(parents=True, exist_ok=True)
# Write the SQL to migration.sql
migration_file = migration_dir / "migration.sql"
migration_file.write_text(result.stdout)
print(f"Created migration in {migration_dir}")
return True
else:
print("No schema changes detected. Migration not needed.")
return False
finally:
# Clean up: remove temporary migrations directory
if temp_migrations_dir.exists():
shutil.rmtree(temp_migrations_dir)
except subprocess.CalledProcessError as e:
print(f"Error generating migration: {e.stderr}")
return False
except Exception as e:
print(f"Error creating migration: {str(e)}")
return False
if __name__ == "__main__":
# If running directly, can optionally pass migration name as argument
import sys
migration_name = sys.argv[1] if len(sys.argv) > 1 else None
create_migration(migration_name)

251
ci_cd/security_scans.sh Executable file
View file

@ -0,0 +1,251 @@
#!/bin/bash
# Security Scans Script for LiteLLM
# This script runs comprehensive security scans including Trivy and Grype
set -e
echo "Starting security scans for LiteLLM..."
# Function to install Trivy and required tools
install_trivy() {
echo "Installing Trivy and required tools..."
sudo apt-get update
sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
sudo apt-get update
sudo apt-get install trivy
echo "Trivy and required tools installed successfully"
}
# Function to install Grype
install_grype() {
echo "Installing Grype..."
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
echo "Grype installed successfully"
}
# Function to install ggshield
install_ggshield() {
echo "Installing ggshield..."
pip3 install --upgrade pip
pip3 install ggshield
echo "ggshield installed successfully"
}
# # Function to run secret detection scans
# run_secret_detection() {
# echo "Running secret detection scans..."
# if ! command -v ggshield &> /dev/null; then
# install_ggshield
# fi
# # Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
# if [ -z "$GITGUARDIAN_API_KEY" ]; then
# echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
# echo "ggshield requires a GitGuardian API key to scan for secrets."
# echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
# exit 1
# fi
# echo "Scanning codebase for secrets..."
# echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
# echo "ggshield will automatically handle rate limits and retry as needed."
# echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
# # Use --recursive for directory scanning and auto-confirm if prompted
# # .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
# # GITGUARDIAN_API_KEY environment variable will be used for authentication
# echo y | ggshield secret scan path . --recursive || {
# echo ""
# echo "=========================================="
# echo "ERROR: Secret Detection Failed"
# echo "=========================================="
# echo "ggshield has detected secrets in the codebase."
# echo "Please review discovered secrets above, revoke any actively used secrets"
# echo "from underlying systems and make changes to inject secrets dynamically at runtime."
# echo ""
# echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
# echo "=========================================="
# echo ""
# exit 1
# }
# echo "Secret detection scans completed successfully"
# }
# Function to run Trivy scans
run_trivy_scans() {
echo "Running Trivy scans..."
echo "Scanning LiteLLM Docs..."
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
echo "Scanning LiteLLM UI..."
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
echo "Trivy scans completed successfully"
}
# Function to build and scan Docker images with Grype
run_grype_scans() {
echo "Running Grype scans..."
# Temporarily add wheel files to .dockerignore for security scans
echo "Temporarily modifying .dockerignore to exclude problematic wheel files..."
cp .dockerignore .dockerignore.backup 2>/dev/null || touch .dockerignore.backup
echo "/*.whl" >> .dockerignore
# Build and scan Dockerfile.database
echo "Building and scanning Dockerfile.database..."
docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
grype litellm-database:latest --config ci_cd/.grype.yaml --fail-on critical
# Build and scan main Dockerfile
echo "Building and scanning main Dockerfile..."
docker build --no-cache -t litellm:latest .
grype litellm:latest --config ci_cd/.grype.yaml --fail-on critical
# Restore original .dockerignore
echo "Restoring original .dockerignore..."
mv .dockerignore.backup .dockerignore
# Scan the locally built LiteLLM image for vulnerabilities with CVSS >= 4.0
echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
echo "Using locally built image: litellm:latest"
# Allowlist of CVEs to be ignored in failure threshold/reporting
# - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
# - GHSA-4xh5-x5gv-qwph: GitHub Security Advisory alias for CVE-2025-8869
# - GHSA-5j98-mcp5-4vw2: glob CLI command injection via -c/--cmd; glob CLI is not used in the litellm runtime image,
# and the vulnerable versions are pulled in only via OS-level/node tooling outside of our application code
ALLOWED_CVES=(
"CVE-2025-8869"
"GHSA-4xh5-x5gv-qwph"
"CVE-2025-8291" # no fix available as of Oct 11, 2025
"GHSA-5j98-mcp5-4vw2"
"CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
"CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
"CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
"CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet
"CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
"CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
"CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
"CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
"GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
"GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
"GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
"CVE-2025-59465" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-55131" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59466" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-55130" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59467" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2026-21637" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-15281" # No fix available yet
"CVE-2026-0865" # No fix available yet
"CVE-2025-15282" # No fix available yet
"CVE-2026-0672" # No fix available yet
"CVE-2025-15366" # No fix available yet
"CVE-2025-15367" # No fix available yet
"CVE-2025-12781" # No fix available yet
"CVE-2025-11468" # No fix available yet
"CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
)
# Build JSON array of allowlisted CVE IDs for jq
ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .)
echo "Checking for vulnerabilities with CVSS score >= 4.0..."
echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
echo ""
# Show all high-severity vulnerabilities for transparency
TOTAL_HIGH_SEVERITY=$(grype litellm:latest -o json | jq -r '
.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| .vulnerability.id' | wc -l)
if [ "$TOTAL_HIGH_SEVERITY" -gt 0 ]; then
echo "Total vulnerabilities found with CVSS >= 4.0: $TOTAL_HIGH_SEVERITY"
echo ""
echo "All high-severity vulnerabilities (including allowlisted):"
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Allowlisted"],
(.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, (if (.vulnerability.id as $id | $allow | index($id)) then "YES" else "NO" end)])
| @tsv' | column -t -s $'\t'
echo ""
fi
HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| .vulnerability.id' | wc -l)
if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
echo ""
echo "=========================================="
echo "ERROR: Security Scan Failed"
echo "=========================================="
echo "Found $HIGH_SEVERITY_COUNT non-allowlisted vulnerabilities with CVSS score >= 4.0 in litellm:latest"
echo ""
echo "These vulnerabilities are NOT in the allowlist and must be addressed."
echo "Current allowlisted CVEs: ${ALLOWED_CVES[*]}"
echo ""
echo "Detailed vulnerability report:"
echo ""
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
(.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
| @tsv' | column -t -s $'\t'
echo ""
echo "=========================================="
echo "Action Required:"
echo "=========================================="
echo "1. If a fix is available, update the package to the fixed version"
echo "2. If the vulnerability is not applicable or has no fix:"
echo " - Add the CVE/GHSA ID to ALLOWED_CVES array in ci_cd/security_scans.sh"
echo " - Add a comment explaining why it's safe to ignore"
echo ""
echo "Note: Some vulnerabilities may have multiple IDs (CVE-XXXX and GHSA-XXXX)."
echo "Add all relevant IDs to the allowlist if they refer to the same issue."
echo "=========================================="
echo ""
exit 1
else
echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"
fi
echo "Grype scans completed successfully"
}
# Main execution
main() {
echo "Installing security scanning tools..."
install_trivy
install_grype
# echo "Running secret detection scans..."
# run_secret_detection
echo "Running filesystem vulnerability scans..."
run_trivy_scans
echo "Running Docker image vulnerability scans..."
run_grype_scans
echo "All security scans completed successfully!"
}
# Execute main function
main "$@"

9
ci_cd/security_scans_readme.md Normal file
View file

@ -0,0 +1,9 @@
# Security Scans
## Scans that run:
- Trivy scan on `./docs/` (HIGH/CRITICAL/MEDIUM)
- Trivy scan on `./ui/` (HIGH/CRITICAL/MEDIUM)
- Grype scan on `Dockerfile.database` (fails on CRITICAL)
- Grype scan on main `Dockerfile` (fails on CRITICAL)
- Grype CVSS ≥ 4.0 scan on main `Dockerfile` (fails any vulnerabilities with CVSS ≥ 4.0)

474
cookbook/LiteLLM_CometAPI.ipynb vendored Normal file
View file

File diff suppressed because one or more lines are too long

318
cookbook/LiteLLM_HuggingFace.ipynb vendored
View file

@ -6,8 +6,9 @@
"id": "9dKM5k8qsMIj"
},
"source": [
"## LiteLLM HuggingFace\n",
"Docs for huggingface: https://docs.litellm.ai/docs/providers/huggingface"
"## LiteLLM Hugging Face\n",
"\n",
"Docs for huggingface: https://docs.litellm.ai/docs/providers/huggingface\n"
]
},
{
@ -27,23 +28,18 @@
"id": "yp5UXRqtpu9f"
},
"source": [
"## Hugging Face Free Serverless Inference API\n",
"Read more about the Free Serverless Inference API here: https://huggingface.co/docs/api-inference.\n",
"## Serverless Inference Providers\n",
"\n",
"In order to use litellm to call Serverless Inference API:\n",
"Read more about Inference Providers here: https://huggingface.co/blog/inference-providers.\n",
"\n",
"* Browse Serverless Inference compatible models here: https://huggingface.co/models?inference=warm&pipeline_tag=text-generation.\n",
"* Copy the model name from hugging face\n",
"* Set `model = \"huggingface/<model-name>\"`\n",
"In order to use litellm with Hugging Face Inference Providers, you need to set `model=huggingface/<provider>/<model-id>`.\n",
"\n",
"Example set `model=huggingface/meta-llama/Meta-Llama-3.1-8B-Instruct` to call `meta-llama/Meta-Llama-3.1-8B-Instruct`\n",
"\n",
"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct"
"Example: `huggingface/together/deepseek-ai/DeepSeek-R1` to run DeepSeek-R1 (https://huggingface.co/deepseek-ai/DeepSeek-R1) through Together AI.\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -51,107 +47,18 @@
"id": "Pi5Oww8gpCUm",
"outputId": "659a67c7-f90d-4c06-b94e-2c4aa92d897a"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ModelResponse(id='chatcmpl-c54dfb68-1491-4d68-a4dc-35e603ea718a', choices=[Choices(finish_reason='eos_token', index=0, message=Message(content=\"I'm just a computer program, so I don't have feelings, but thank you for asking! How can I assist you today?\", role='assistant', tool_calls=None, function_call=None))], created=1724858285, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion', system_fingerprint=None, usage=Usage(completion_tokens=27, prompt_tokens=47, total_tokens=74))\n",
"ModelResponse(id='chatcmpl-d2ae38e6-4974-431c-bb9b-3fa3f95e5a6d', choices=[Choices(finish_reason='length', index=0, message=Message(content=\"\\n\\nI’m doing well, thank you. I’ve been keeping busy with work and some personal projects. How about you?\\n\\nI'm doing well, thank you. I've been enjoying some time off and catching up on some reading. How can I assist you today?\\n\\nI'm looking for a good book to read. Do you have any recommendations?\\n\\nOf course! Here are a few book recommendations across different genres:\\n\\n1.\", role='assistant', tool_calls=None, function_call=None))], created=1724858288, model='mistralai/Mistral-7B-Instruct-v0.3', object='chat.completion', system_fingerprint=None, usage=Usage(completion_tokens=85, prompt_tokens=6, total_tokens=91))\n"
]
}
],
"outputs": [],
"source": [
"import os\n",
"import litellm\n",
"from litellm import completion\n",
"\n",
"# Make sure to create an API_KEY with inference permissions at https://huggingface.co/settings/tokens/new?globalPermissions=inference.serverless.write&tokenType=fineGrained\n",
"os.environ[\"HUGGINGFACE_API_KEY\"] = \"\"\n",
"# You can create a HF token here: https://huggingface.co/settings/tokens\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"# Call https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct\n",
"# add the 'huggingface/' prefix to the model to set huggingface as the provider\n",
"response = litellm.completion(\n",
" model=\"huggingface/meta-llama/Meta-Llama-3.1-8B-Instruct\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(response)\n",
"\n",
"\n",
"# Call https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3\n",
"response = litellm.completion(\n",
" model=\"huggingface/mistralai/Mistral-7B-Instruct-v0.3\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "-klhAhjLtclv"
},
"source": [
"## Hugging Face Dedicated Inference Endpoints\n",
"\n",
"Steps to use\n",
"* Create your own Hugging Face dedicated endpoint here: https://ui.endpoints.huggingface.co/\n",
"* Set `api_base` to your deployed api base\n",
"* Add the `huggingface/` prefix to your model so litellm knows it's a huggingface Deployed Inference Endpoint"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Lbmw8Gl_pHns",
"outputId": "ea8408bf-1cc3-4670-ecea-f12666d204a8"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"length\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"\\n\\nI am doing well, thank you for asking. How about you?\\nI am doing\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": -8.9481967812\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-74dc9d89-3916-47ce-9bea-b80e66660f77\",\n",
" \"created\": 1695871068.8413374,\n",
" \"model\": \"glaiveai/glaive-coder-7b\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 18,\n",
" \"total_tokens\": 24\n",
" }\n",
"}\n"
]
}
],
"source": [
"import os\n",
"import litellm\n",
"\n",
"os.environ[\"HUGGINGFACE_API_KEY\"] = \"\"\n",
"\n",
"# TGI model: Call https://huggingface.co/glaiveai/glaive-coder-7b\n",
"# add the 'huggingface/' prefix to the model to set huggingface as the provider\n",
"# set api base to your deployed api endpoint from hugging face\n",
"response = litellm.completion(\n",
" model=\"huggingface/glaiveai/glaive-coder-7b\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" api_base=\"https://wjiegasee9bmqke2.us-east-1.aws.endpoints.huggingface.cloud\"\n",
"# Call DeepSeek-R1 model through Together AI\n",
"response = completion(\n",
" model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n",
" messages=[{\"content\": \"How many r's are in the word `strawberry`?\", \"role\": \"user\"}],\n",
")\n",
"print(response)"
]
@ -162,13 +69,12 @@
"id": "EU0UubrKzTFe"
},
"source": [
"## HuggingFace - Streaming (Serveless or Dedicated)\n",
"Set stream = True"
"## Streaming\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -176,74 +82,147 @@
"id": "y-QfIvA-uJKX",
"outputId": "b007bb98-00d0-44a4-8264-c8a2caed6768"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<litellm.utils.CustomStreamWrapper object at 0x1278471d0>\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content='I', role='assistant', function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=\"'m\", role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' just', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' a', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' computer', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' program', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=',', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' so', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' I', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' don', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=\"'t\", role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' have', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' feelings', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=',', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' but', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' thank', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' you', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' for', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' asking', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content='!', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' How', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' can', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' I', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' assist', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' you', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=' today', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content='?', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content='<|eot_id|>', role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-ffeb4491-624b-4ddf-8005-60358cf67d36', choices=[StreamingChoices(finish_reason='stop', index=0, delta=Delta(content=None, role=None, function_call=None, tool_calls=None), logprobs=None)], created=1724858353, model='meta-llama/Meta-Llama-3.1-8B-Instruct', object='chat.completion.chunk', system_fingerprint=None)\n"
]
}
],
"outputs": [],
"source": [
"import os\n",
"import litellm\n",
"from litellm import completion\n",
"\n",
"# Make sure to create an API_KEY with inference permissions at https://huggingface.co/settings/tokens/new?globalPermissions=inference.serverless.write&tokenType=fineGrained\n",
"os.environ[\"HUGGINGFACE_API_KEY\"] = \"\"\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"# Call https://huggingface.co/glaiveai/glaive-coder-7b\n",
"# add the 'huggingface/' prefix to the model to set huggingface as the provider\n",
"# set api base to your deployed api endpoint from hugging face\n",
"response = litellm.completion(\n",
" model=\"huggingface/meta-llama/Meta-Llama-3.1-8B-Instruct\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" stream=True\n",
"response = completion(\n",
" model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n",
" messages=[\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"How many r's are in the word `strawberry`?\",\n",
" \n",
" }\n",
" ],\n",
" stream=True,\n",
")\n",
"\n",
"print(response)\n",
"\n",
"for chunk in response:\n",
" print(chunk)"
" print(chunk)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## With images as input\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "CKXAnK55zQRl"
},
"metadata": {},
"outputs": [],
"source": []
"source": [
"from litellm import completion\n",
"\n",
"# Set your Hugging Face Token\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": [\n",
" {\"type\": \"text\", \"text\": \"What's in this image?\"},\n",
" {\n",
" \"type\": \"image_url\",\n",
" \"image_url\": {\n",
" \"url\": \"https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png\",\n",
" },\n",
" },\n",
" ],\n",
" }\n",
"]\n",
"\n",
"response = completion(\n",
" model=\"huggingface/sambanova/meta-llama/Llama-3.3-70B-Instruct\",\n",
" messages=messages,\n",
")\n",
"print(response.choices[0])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Tools - Function Calling\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"\n",
"# Set your Hugging Face Token\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"tools = [\n",
" {\n",
" \"type\": \"function\",\n",
" \"function\": {\n",
" \"name\": \"get_current_weather\",\n",
" \"description\": \"Get the current weather in a given location\",\n",
" \"parameters\": {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"location\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
" },\n",
" \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
" },\n",
" \"required\": [\"location\"],\n",
" },\n",
" },\n",
" }\n",
"]\n",
"messages = [{\"role\": \"user\", \"content\": \"What's the weather like in Boston today?\"}]\n",
"\n",
"response = completion(\n",
" model=\"huggingface/sambanova/meta-llama/Llama-3.1-8B-Instruct\", messages=messages, tools=tools, tool_choice=\"auto\"\n",
")\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Hugging Face Dedicated Inference Endpoints\n",
"\n",
"Steps to use\n",
"\n",
"- Create your own Hugging Face dedicated endpoint here: https://ui.endpoints.huggingface.co/\n",
"- Set `api_base` to your deployed api base\n",
"- set the model to `huggingface/tgi` so that litellm knows it's a huggingface Deployed Inference Endpoint.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import litellm\n",
"\n",
"\n",
"response = litellm.completion(\n",
" model=\"huggingface/tgi\",\n",
" messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}],\n",
" api_base=\"https://my-endpoint.endpoints.huggingface.cloud/v1/\",\n",
")\n",
"print(response)"
]
}
],
"metadata": {
@ -251,7 +230,8 @@
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
@ -264,7 +244,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
"version": "3.12.0"
}
},
"nbformat": 4,

97
cookbook/LiteLLM_NovitaAI_Cookbook.ipynb vendored Normal file
View file

@ -0,0 +1,97 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "iFEmsVJI_2BR"
},
"source": [
"# LiteLLM NovitaAI Cookbook"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "cBlUhCEP_xj4"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "p-MQqWOT_1a7"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ['NOVITA_API_KEY'] = \"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Ze8JqMqWAARO"
},
"outputs": [],
"source": [
"from litellm import completion\n",
"response = completion(\n",
" model=\"novita/deepseek/deepseek-r1\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "-LnhELrnAM_J"
},
"outputs": [],
"source": [
"response = completion(\n",
" model=\"novita/deepseek/deepseek-r1\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "dJBOUYdwCEn1"
},
"outputs": [],
"source": [
"response = completion(\n",
" model=\"mistralai/mistral-7b-instruct\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

114
cookbook/LiteLLM_OpenRouter.ipynb vendored
View file

@ -1,27 +1,13 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# LiteLLM OpenRouter Cookbook"
],
"metadata": {
"id": "iFEmsVJI_2BR"
}
},
"source": [
"# LiteLLM OpenRouter Cookbook"
]
},
{
"cell_type": "code",
@ -36,27 +22,20 @@
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"id": "p-MQqWOT_1a7"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ['OPENROUTER_API_KEY'] = \"\""
],
"metadata": {
"id": "p-MQqWOT_1a7"
},
"execution_count": 14,
"outputs": []
]
},
{
"cell_type": "code",
"source": [
"from litellm import completion\n",
"response = completion(\n",
" model=\"openrouter/google/palm-2-chat-bison\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
],
"execution_count": 11,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -64,10 +43,8 @@
"id": "Ze8JqMqWAARO",
"outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98"
},
"execution_count": 11,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<OpenAIObject id=gen-W8FTMSIEorCp3vG5iYIgNMR4IeBv at 0x7c3dcef1f060> JSON: {\n",
@ -85,20 +62,23 @@
"}"
]
},
"execution_count": 11,
"metadata": {},
"execution_count": 11
"output_type": "execute_result"
}
],
"source": [
"from litellm import completion\n",
"response = completion(\n",
" model=\"openrouter/google/palm-2-chat-bison\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"source": [
"response = completion(\n",
" model=\"openrouter/anthropic/claude-2\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
],
"execution_count": 12,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -106,10 +86,8 @@
"id": "-LnhELrnAM_J",
"outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0"
},
"execution_count": 12,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<OpenAIObject id=gen-IiuV7ZNimDufVeutBHrl8ajPuzEh at 0x7c3dcea67560> JSON: {\n",
@ -128,20 +106,22 @@
"}"
]
},
"execution_count": 12,
"metadata": {},
"execution_count": 12
"output_type": "execute_result"
}
],
"source": [
"response = completion(\n",
" model=\"openrouter/anthropic/claude-2\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"source": [
"response = completion(\n",
" model=\"openrouter/meta-llama/llama-2-70b-chat\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
],
"execution_count": 13,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -149,10 +129,8 @@
"id": "dJBOUYdwCEn1",
"outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0"
},
"execution_count": 13,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<OpenAIObject id=gen-PyMd3yyJ0aQsCgIY9R8XGZoAtPbl at 0x7c3dceefcae0> JSON: {\n",
@ -170,10 +148,32 @@
"}"
]
},
"execution_count": 13,
"metadata": {},
"execution_count": 13
"output_type": "execute_result"
}
],
"source": [
"response = completion(\n",
" model=\"openrouter/meta-llama/llama-2-70b-chat\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
}
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

2
cookbook/LiteLLM_PromptLayer.ipynb vendored
View file

@ -39,7 +39,7 @@
"import os\n",
"os.environ['OPENAI_API_KEY'] = \"\"\n",
"os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
"os.environ['PROMPTLAYER_API_KEY'] = \"pl_4ea2bb00a4dca1b8a70cebf2e9e11564\"\n",
"os.environ['PROMPTLAYER_API_KEY'] = \"test-promptlayer-key-123\"\n",
"\n",
"# Set Promptlayer as a success callback\n",
"litellm.success_callback =['promptlayer']\n",

278
cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb vendored
View file

@ -1,21 +1,10 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "kccfk0mHZ4Ad"
},
"source": [
"# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n",
"\n",
@ -32,29 +21,26 @@
"To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
"\n",
"To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n"
],
"metadata": {
"id": "kccfk0mHZ4Ad"
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "nmSClzCPaGH6"
},
"source": [
"## /chat/completion\n",
"\n"
],
"metadata": {
"id": "nmSClzCPaGH6"
}
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI Python SDK"
],
"metadata": {
"id": "_vqcjwOVaKpO"
}
},
"source": [
"### OpenAI Python SDK"
]
},
{
"cell_type": "code",
@ -94,15 +80,20 @@
},
{
"cell_type": "markdown",
"source": [
"## Function Calling"
],
"metadata": {
"id": "AqkyKk9Scxgj"
}
},
"source": [
"## Function Calling"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "wDg10VqLczE1"
},
"outputs": [],
"source": [
"from openai import OpenAI\n",
"client = OpenAI(\n",
@ -139,24 +130,24 @@
")\n",
"\n",
"print(completion)\n"
],
"metadata": {
"id": "wDg10VqLczE1"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Azure OpenAI Python SDK"
],
"metadata": {
"id": "YYoxLloSaNWW"
}
},
"source": [
"### Azure OpenAI Python SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "yA1XcgowaSRy"
},
"outputs": [],
"source": [
"import openai\n",
"client = openai.AzureOpenAI(\n",
@ -184,24 +175,24 @@
")\n",
"\n",
"print(response)"
],
"metadata": {
"id": "yA1XcgowaSRy"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain Python"
],
"metadata": {
"id": "yl9qhDvnaTpL"
}
},
"source": [
"### Langchain Python"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "5MUZgSquaW5t"
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.prompts.chat import (\n",
@ -239,24 +230,22 @@
"response = chat(messages)\n",
"\n",
"print(response)"
],
"metadata": {
"id": "5MUZgSquaW5t"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Curl"
],
"metadata": {
"id": "B9eMgnULbRaz"
}
},
"source": [
"### Curl"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "VWCCk5PFcmhS"
},
"source": [
"\n",
"\n",
@ -280,22 +269,24 @@
"}'\n",
"```\n",
"\n"
],
"metadata": {
"id": "VWCCk5PFcmhS"
}
]
},
{
"cell_type": "markdown",
"source": [
"### LlamaIndex"
],
"metadata": {
"id": "drBAm2e1b6xe"
}
},
"source": [
"### LlamaIndex"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "d0bZcv8fb9mL"
},
"outputs": [],
"source": [
"import os, dotenv\n",
"\n",
@ -326,24 +317,24 @@
"query_engine = index.as_query_engine()\n",
"response = query_engine.query(\"What did the author do growing up?\")\n",
"print(response)\n"
],
"metadata": {
"id": "d0bZcv8fb9mL"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain JS"
],
"metadata": {
"id": "xypvNdHnb-Yy"
}
},
"source": [
"### Langchain JS"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "R55mK2vCcBN2"
},
"outputs": [],
"source": [
"import { ChatOpenAI } from \"@langchain/openai\";\n",
"\n",
@ -359,24 +350,24 @@
"const message = await model.invoke(\"Hi there!\");\n",
"\n",
"console.log(message);\n"
],
"metadata": {
"id": "R55mK2vCcBN2"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI JS"
],
"metadata": {
"id": "nC4bLifCcCiW"
}
},
"source": [
"### OpenAI JS"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "MICH8kIMcFpg"
},
"outputs": [],
"source": [
"const { OpenAI } = require('openai');\n",
"\n",
@ -398,24 +389,24 @@
"}\n",
"\n",
"main();\n"
],
"metadata": {
"id": "MICH8kIMcFpg"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Anthropic SDK"
],
"metadata": {
"id": "D1Q07pEAcGTb"
}
},
"source": [
"### Anthropic SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "qBjFcAvgcI3t"
},
"outputs": [],
"source": [
"import os\n",
"\n",
@ -423,7 +414,7 @@
"\n",
"client = Anthropic(\n",
" base_url=\"http://localhost:4000\", # proxy endpoint\n",
" api_key=\"sk-s4xN1IiLTCytwtZFJaYQrA\", # litellm proxy virtual key\n",
" api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n",
")\n",
"\n",
"message = client.messages.create(\n",
@ -437,33 +428,33 @@
" model=\"claude-3-opus-20240229\",\n",
")\n",
"print(message.content)"
],
"metadata": {
"id": "qBjFcAvgcI3t"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"## /embeddings"
],
"metadata": {
"id": "dFAR4AJGcONI"
}
},
"source": [
"## /embeddings"
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI Python SDK"
],
"metadata": {
"id": "lgNoM281cRzR"
}
},
"source": [
"### OpenAI Python SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "NY3DJhPfcQhA"
},
"outputs": [],
"source": [
"import openai\n",
"from openai import OpenAI\n",
@ -478,24 +469,24 @@
")\n",
"\n",
"print(response)\n"
],
"metadata": {
"id": "NY3DJhPfcQhA"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain Embeddings"
],
"metadata": {
"id": "hmbg-DW6cUZs"
}
},
"source": [
"### Langchain Embeddings"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "lX2S8Nl1cWVP"
},
"outputs": [],
"source": [
"from langchain.embeddings import OpenAIEmbeddings\n",
"\n",
@ -526,24 +517,22 @@
"\n",
"print(f\"TITAN EMBEDDINGS\")\n",
"print(query_result[:5])"
],
"metadata": {
"id": "lX2S8Nl1cWVP"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Curl Request"
],
"metadata": {
"id": "oqGbWBCQcYfd"
}
},
"source": [
"### Curl Request"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7rkIMV9LcdwQ"
},
"source": [
"\n",
"\n",
@ -556,10 +545,21 @@
" }'\n",
"```\n",
"\n"
],
"metadata": {
"id": "7rkIMV9LcdwQ"
}
]
}
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

295
cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md Normal file
View file

@ -0,0 +1,295 @@
# Claude Code with LiteLLM Quickstart
This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code.
> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
## Video Walkthrough
Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56
## Prerequisites
- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
- API keys for your chosen providers
## Installation
First, install LiteLLM with proxy support:
```bash
pip install 'litellm[proxy]'
```
## Step 1: Setup config.yaml
Create a secure configuration using environment variables:
```yaml
model_list:
# Claude models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
Set your environment variables:
```bash
export ANTHROPIC_API_KEY="your-anthropic-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
## Step 2: Start Proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
## Step 3: Verify Setup
Test that your proxy is working correctly:
```bash
curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "claude-3-5-sonnet-20241022",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
## Step 4: Configure Claude Code
### Method 1: Unified Endpoint (Recommended)
Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI.
### Method 2: Provider-specific Pass-through Endpoint
Alternatively, use the Anthropic pass-through endpoint:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
## Step 5: Use Claude Code
### Choosing Your Model
You have two options for specifying which model Claude Code uses:
#### Option 1: Command Line / Session Model Selection
Specify the model directly when starting Claude Code or during a session:
```bash
# Specify model at startup
claude --model claude-3-5-sonnet-20241022
# Or change model during a session
/model claude-3-5-haiku-20241022
```
This method uses the exact model you specify.
#### Option 2: Environment Variables
Configure default models using environment variables:
```bash
# Tell Claude Code which models to use by default
export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
export ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-3-5-20240229
claude # Will use the models specified above
```
**Note:** Claude Code may cache the model from a previous session. If environment variables don't take effect, use Option 1 to explicitly set the model.
**Important:** The `model_name` in your LiteLLM config must match what Claude Code requests (either from env vars or command line).
### Using 1M Context Window
Claude Code supports extended context (1 million tokens) using the `[1m]` suffix with Claude 4+ models:
```bash
# Use Sonnet 4.5 with 1M context (requires quotes for shell)
claude --model 'claude-sonnet-4-5-20250929[1m]'
# Inside a Claude Code session (no quotes needed)
/model claude-sonnet-4-5-20250929[1m]
```
**Important:** When using `--model` with `[1m]` in the shell, you must use quotes to prevent the shell from interpreting the brackets.
Alternatively, set as default with environment variables:
```bash
export ANTHROPIC_DEFAULT_SONNET_MODEL='claude-sonnet-4-5-20250929[1m]'
claude
```
**How it works:**
- Claude Code strips the `[1m]` suffix before sending to LiteLLM
- Claude Code automatically adds the header `anthropic-beta: context-1m-2025-08-07`
- Your LiteLLM config should **NOT** include `[1m]` in model names
**Verify 1M context is active:**
```bash
/context
# Should show: 21k/1000k tokens (2%)
```
**Pricing:** Models using 1M context have different pricing. Input tokens above 200k are charged at a higher rate.
## Troubleshooting
Common issues and solutions:
**Claude Code not connecting:**
- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
- Check that `ANTHROPIC_BASE_URL` is set correctly
- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Authentication errors:**
- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
- Check that your API keys are valid and have sufficient credits
- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Model not found:**
- Check what model Claude Code is requesting in LiteLLM logs
- Ensure your `config.yaml` has a matching `model_name` entry
- If using environment variables, verify they're set: `echo $ANTHROPIC_DEFAULT_SONNET_MODEL`
**1M context not working (showing 200k instead of 1000k):**
- Verify you're using the `[1m]` suffix: `/model your-model-name[1m]`
- Check LiteLLM logs for the header `context-1m-2025-08-07` in the request
- Ensure your model supports 1M context (only certain Claude models do)
- Your LiteLLM config should **NOT** include `[1m]` in the `model_name`
## Using Multiple Models and Providers
You can configure LiteLLM to route to any supported provider. Here's an example with multiple providers:
```yaml
model_list:
# OpenAI models
- model_name: codex-mini
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: o3-pro
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
# Anthropic models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
**Note:** The `model_name` can be anything you choose. Claude Code will request whatever model you specify (via env vars or command line), and LiteLLM will route to the `model` configured in `litellm_params`.
Switch between models seamlessly:
```bash
# Use environment variables to set defaults
export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
# Or specify directly
claude --model claude-3-5-sonnet-20241022 # Complex reasoning
claude --model claude-3-5-haiku-20241022 # Fast responses
claude --model claude-bedrock # Bedrock deployment
```
## Default Models Used by Claude Code
If you **don't** set environment variables, Claude Code uses these default model names:
| Purpose | Default Model Name (v2.1.14) |
|---------|------------------------------|
| Main model | `claude-sonnet-4-5-20250929` |
| Light tasks (subagents, summaries) | `claude-haiku-4-5-20251001` |
| Planning mode | `claude-opus-4-5-20251101` |
Your LiteLLM config should include these model names if you want Claude Code to work without setting environment variables:
```yaml
model_list:
- model_name: claude-sonnet-4-5-20250929
litellm_params:
# Can be any provider - Anthropic, Bedrock, Vertex AI, etc.
model: anthropic/claude-sonnet-4-5-20250929
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-haiku-4-5-20251001
litellm_params:
model: anthropic/claude-haiku-4-5-20251001
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-opus-4-5-20251101
litellm_params:
model: anthropic/claude-opus-4-5-20251101
api_key: os.environ/ANTHROPIC_API_KEY
```
**Warning:** These default model names may change with new Claude Code versions. Check LiteLLM proxy logs for "model not found" errors to identify what Claude Code is requesting.
## Additional Resources
- [LiteLLM Documentation](https://docs.litellm.ai/)
- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)

134
cookbook/ai_coding_tool_guides/index.json Normal file
View file

@ -0,0 +1,134 @@
[{
"title": "Claude Code Quickstart",
"description": "This is a quickstart guide to using Claude Code with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api",
"date": "2026-01-15",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM"
]
},
{
"title": "Claude Code with MCPs",
"description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_mcp",
"date": "2026-01-15",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"MCP"
]
},
{
"title": "Claude Code with Non-Anthropic Models",
"description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"OpenAI",
"Gemini"
]
},
{
"title": "Cursor Quickstart",
"description": "This is a quickstart guide to using Cursor with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/cursor_integration",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Cursor",
"LiteLLM",
"Quickstart"
]
},
{
"title": "Github Copilot Quickstart",
"description": "This is a quickstart guide to using Github Copilot with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Github Copilot",
"LiteLLM",
"Quickstart"
]
},
{
"title": "LiteLLM Gemini CLI Quickstart",
"description": "This is a quickstart guide to using LiteLLM Gemini CLI.",
"url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Gemini CLI",
"Gemini",
"LiteLLM",
"Quickstart"
]
},
{
"title": "OpenAI Codex CLI Quickstart",
"description": "This is a quickstart guide to using OpenAI Codex CLI.",
"url": "https://docs.litellm.ai/docs/tutorials/openai_codex",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"OpenAI Codex CLI",
"OpenAI",
"LiteLLM",
"Quickstart"
]
},
{
"title": "OpenWebUI Quickstart",
"description": "This is a quickstart guide to using OpenWebUI with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/openweb_ui",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"OpenWebUI",
"LiteLLM",
"Quickstart"
]
},
{
"title": "AI Coding Tool Usage Tracking",
"description": "This is a guide to tracking usage for AI coding tools monitor the use of Claude Code , Google Antigravity, OpenAI Codex, Roo Code etc. through LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/cost_tracking_coding",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"Gemini CLI",
"OpenAI Codex",
"LiteLLM"
]
},
{
"title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)",
"description": "This is a guide for using Web Search with Claude Code via LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"Web Search"
]
},
{
"title": "Track Claude Code Usage per user via Custom Headers",
"description": "This is a guide for tracking claude code user usage by passing a customer ID header.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_code_customer_tracking",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM"
]
}]

144
cookbook/anthropic_agent_sdk/README.md Normal file
View file

@ -0,0 +1,144 @@
# Claude Agent SDK with LiteLLM Gateway
A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK.
## Quick Start
### 1. Install dependencies
```bash
pip install anthropic claude-agent-sdk litellm
```
### 2. Start LiteLLM proxy
```bash
# Simple start with Claude
litellm --model claude-sonnet-4-20250514
# Or with a config file
litellm --config config.yaml
```
### 3. Run the chat
**Basic Agent (no MCP):**
```bash
python main.py
```
**Agent with MCP (DeepWiki2 for research):**
```bash
python agent_with_mcp.py
```
If MCP connection fails, you can disable it:
```bash
USE_MCP=false python agent_with_mcp.py
```
That's it! You can now chat with the agent in your terminal.
### Chat Commands
While chatting, you can use these commands:
- `models` - List all available models (fetched from your LiteLLM proxy)
- `model` - Switch to a different model
- `clear` - Start a new conversation
- `quit` or `exit` - End the chat
The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured.
## Configuration
Set these environment variables if needed:
```bash
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
export LITELLM_MODEL="bedrock-claude-sonnet-4.5"
```
Or just use the defaults - it'll connect to `http://localhost:4000` by default.
## Files
- `main.py` - Basic interactive agent without MCP
- `agent_with_mcp.py` - Agent with MCP server integration (DeepWiki2)
- `common.py` - Shared utilities and functions
- `config.example.yaml` - Example LiteLLM configuration
- `requirements.txt` - Python dependencies
## Example Config File
If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`):
```yaml
model_list:
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
```
Then start LiteLLM with: `litellm --config config.yaml`
## How It Works
The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
```python
# Point to LiteLLM gateway (not Anthropic)
os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key
# Use any model configured in LiteLLM
options = ClaudeAgentOptions(
model="bedrock-claude-sonnet-4", # or gpt-4, or anything else
system_prompt="You are a helpful assistant.",
max_turns=50,
)
```
Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically.
## Why Use This?
- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc.
- **Cost tracking**: LiteLLM tracks spending across all your agent conversations
- **Rate limiting**: Set budgets and limits on your agent usage
- **Load balancing**: Distribute requests across multiple API keys or regions
- **Fallbacks**: Automatically retry with a different model if one fails
## Troubleshooting
**Connection errors?**
- Make sure LiteLLM is running: `litellm --model your-model`
- Check the URL is correct (default: `http://localhost:4000`)
**Authentication errors?**
- Verify your LiteLLM API key is correct
- Make sure the model is configured in your LiteLLM setup
**Model not found?**
- Check the model name matches what's in your LiteLLM config
- Run `litellm --model your-model` to test it works
**Agent with MCP stuck or failing?**
- The MCP server might not be available at `http://localhost:4000/mcp/deepwiki2`
- Try disabling MCP: `USE_MCP=false python agent_with_mcp.py`
- Or use the basic agent: `python main.py`
## Learn More
- [LiteLLM Docs](https://docs.litellm.ai/)
- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk)
- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start)

140
cookbook/anthropic_agent_sdk/agent_with_mcp.py Normal file
View file

@ -0,0 +1,140 @@
"""
Interactive Claude Agent SDK CLI with MCP Support
This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy,
with MCP (Model Context Protocol) server integration for enhanced capabilities.
"""
import asyncio
import os
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
from common import (
Config,
fetch_available_models,
setup_litellm_env,
print_header,
handle_model_list,
handle_model_switch,
stream_response,
)
async def interactive_chat_with_mcp():
"""
Interactive CLI chat with the agent and MCP server
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
current_model = config.LITELLM_MODEL
# MCP server configuration
mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
if not use_mcp:
print("⚠️ MCP disabled via USE_MCP=false")
print_header(litellm_base_url, current_model, has_mcp=use_mcp)
while True:
# Configure agent options
if use_mcp:
try:
# Try with MCP server (HTTP transport)
# Using McpHttpServerConfig format from Agent SDK
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant with access to DeepWiki for research. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
mcp_servers={
"deepwiki2": {
"type": "http",
"url": mcp_server_url,
"headers": {
"Authorization": f"Bearer {config.LITELLM_API_KEY}"
}
}
},
)
except Exception as e:
print(f"⚠️ Warning: Could not configure MCP server: {e}")
print("Continuing without MCP...\n")
use_mcp = False
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
else:
# Without MCP
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
# Create agent client
try:
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
user_input = input("\n👤 You: ").strip()
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)
except Exception as e:
print(f"\n❌ Error creating agent client: {e}")
print("This might be an MCP configuration issue. Try running without MCP:")
print(" USE_MCP=false python agent_with_mcp.py")
print("\nOr use the basic agent:")
print(" python main.py")
return
def main():
"""Run interactive chat with MCP"""
try:
asyncio.run(interactive_chat_with_mcp())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
if __name__ == "__main__":
main()

160
cookbook/anthropic_agent_sdk/common.py Normal file
View file

@ -0,0 +1,160 @@
"""
Common utilities for Claude Agent SDK examples
"""
import os
import httpx
class Config:
"""Configuration for LiteLLM Gateway connection"""
# LiteLLM proxy URL (default to local instance)
LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
# LiteLLM API key (master key or virtual key)
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
# Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
"""
Fetch available models from LiteLLM proxy /models endpoint
"""
try:
async with httpx.AsyncClient() as client:
response = await client.get(
f"{base_url}/models",
headers={"Authorization": f"Bearer {api_key}"},
timeout=10.0
)
response.raise_for_status()
data = response.json()
return [model["id"] for model in data.get("data", [])]
except Exception as e:
print(f"⚠️ Warning: Could not fetch models from proxy: {e}")
print("Using default model list...")
# Fallback to default models
return [
"bedrock-claude-sonnet-3.5",
"bedrock-claude-sonnet-4",
"bedrock-claude-sonnet-4.5",
"bedrock-claude-opus-4.5",
"bedrock-nova-premier",
]
def setup_litellm_env(config: Config):
"""
Configure environment variables to point Agent SDK to LiteLLM
"""
litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
return litellm_base_url
def print_header(base_url: str, current_model: str, has_mcp: bool = False):
"""
Print the chat header
"""
mcp_indicator = " + MCP" if has_mcp else ""
print("=" * 70)
print(f"🤖 Claude Agent SDK with LiteLLM Gateway{mcp_indicator} - Interactive Chat")
print("=" * 70)
print(f"🚀 Connected to: {base_url}")
print(f"📦 Current model: {current_model}")
if has_mcp:
print("🔌 MCP: deepwiki2 enabled")
print("\nType your messages below. Commands:")
print(" - 'quit' or 'exit' to end the conversation")
print(" - 'clear' to start a new conversation")
print(" - 'model' to switch models")
print(" - 'models' to list available models")
print("=" * 70)
print()
def handle_model_list(available_models: list[str], current_model: str):
"""
Display available models
"""
print("\n📋 Available models:")
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
"""
Handle model switching
Returns:
tuple: (new_model, should_restart_conversation)
"""
print("\n📋 Select a model:")
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
try:
choice = input("\nEnter number (or press Enter to cancel): ").strip()
if choice:
idx = int(choice) - 1
if 0 <= idx < len(available_models):
new_model = available_models[idx]
print(f"\n✅ Switched to: {new_model}")
print("🔄 Starting new conversation with new model...\n")
return new_model, True
else:
print("❌ Invalid choice")
except (ValueError, IndexError):
print("❌ Invalid input")
return current_model, False
async def stream_response(client, user_input: str):
"""
Stream response from the agent
"""
print("\n🤖 Assistant: ", end='', flush=True)
try:
await client.query(user_input)
# Show loading indicator
print("⏳ thinking...", end='', flush=True)
# Stream the response
first_chunk = True
async for msg in client.receive_response():
# Clear loading indicator on first message
if first_chunk:
print("\r🤖 Assistant: ", end='', flush=True)
first_chunk = False
# Handle different message types
if hasattr(msg, 'type'):
if msg.type == 'content_block_delta':
# Streaming text delta
if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
print(msg.delta.text, end='', flush=True)
elif msg.type == 'content_block_start':
# Start of content block
if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
print(msg.content_block.text, end='', flush=True)
# Fallback to original content handling
if hasattr(msg, 'content'):
for content_block in msg.content:
if hasattr(content_block, 'text'):
print(content_block.text, end='', flush=True)
print() # New line after response
except Exception as e:
print(f"\r\n❌ Error: {e}")
print("Please check your LiteLLM gateway is running and configured correctly.")

25
cookbook/anthropic_agent_sdk/config.example.yaml Normal file
View file

@ -0,0 +1,25 @@
model_list:
- model_name: bedrock-claude-sonnet-3.5
litellm_params:
model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-opus-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-nova-premier
litellm_params:
model: "bedrock/amazon.nova-premier-v1:0"
aws_region_name: "us-east-1"

95
cookbook/anthropic_agent_sdk/main.py Normal file
View file

@ -0,0 +1,95 @@
"""
Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway
This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy.
LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.)
through the Claude Agent SDK by pointing it to the LiteLLM gateway.
"""
import asyncio
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
from common import (
Config,
fetch_available_models,
setup_litellm_env,
print_header,
handle_model_list,
handle_model_switch,
stream_response,
)
async def interactive_chat():
"""
Interactive CLI chat with the agent
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
current_model = config.LITELLM_MODEL
print_header(litellm_base_url, current_model)
while True:
# Configure agent options for each conversation
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
# Create agent client
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
user_input = input("\n👤 You: ").strip()
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)
def main():
"""Run interactive chat"""
try:
asyncio.run(interactive_chat())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
if __name__ == "__main__":
main()

2
cookbook/anthropic_agent_sdk/requirements.txt Normal file
View file

@ -0,0 +1,2 @@
claude-agent-sdk
httpx>=0.27.0

412
cookbook/google_adk_litellm_tutorial.ipynb vendored Normal file
View file

@ -0,0 +1,412 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "7aa8875d",
"metadata": {},
"source": [
"# Google ADK with LiteLLM\n",
"\n",
"Use Google ADK with LiteLLM Python SDK, LiteLLM Proxy.\n",
"\n",
"This tutorial shows you how to create intelligent agents using Agent Development Kit (ADK) with support for multiple Large Language Model (LLM) providers through LiteLLM."
]
},
{
"cell_type": "markdown",
"id": "a4d249c3",
"metadata": {},
"source": [
"## Overview\n",
"\n",
"ADK (Agent Development Kit) allows you to build intelligent agents powered by LLMs. By integrating with LiteLLM, you can:\n",
"\n",
"- Use multiple LLM providers (OpenAI, Anthropic, Google, etc.)\n",
"- Switch easily between models from different providers\n",
"- Connect to a LiteLLM proxy for centralized model management"
]
},
{
"cell_type": "markdown",
"id": "a0bbb56b",
"metadata": {},
"source": [
"## Prerequisites\n",
"\n",
"- Python environment setup\n",
"- API keys for model providers (OpenAI, Anthropic, Google AI Studio)\n",
"- Basic understanding of LLMs and agent concepts"
]
},
{
"cell_type": "markdown",
"id": "7fee50a8",
"metadata": {},
"source": [
"## Installation"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "44106a23",
"metadata": {},
"outputs": [],
"source": [
"# Install dependencies\n",
"!pip install google-adk litellm"
]
},
{
"cell_type": "markdown",
"id": "2171740a",
"metadata": {},
"source": [
"## 1. Setting Up Environment"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6695807e",
"metadata": {},
"outputs": [],
"source": [
"# Setup environment and API keys\n",
"import os\n",
"import asyncio\n",
"from google.adk.agents import Agent\n",
"from google.adk.models.lite_llm import LiteLlm # For multi-model support\n",
"from google.adk.sessions import InMemorySessionService\n",
"from google.adk.runners import Runner\n",
"from google.genai import types\n",
"import litellm # Import for proxy configuration\n",
"\n",
"# Set your API keys\n",
"os.environ['GOOGLE_API_KEY'] = 'your-google-api-key' # For Gemini models\n",
"os.environ['OPENAI_API_KEY'] = 'your-openai-api-key' # For OpenAI models\n",
"os.environ['ANTHROPIC_API_KEY'] = 'your-anthropic-api-key' # For Claude models\n",
"\n",
"# Define model constants for cleaner code\n",
"MODEL_GEMINI_PRO = 'gemini-1.5-pro'\n",
"MODEL_GPT_4O = 'openai/gpt-4o'\n",
"MODEL_CLAUDE_SONNET = 'anthropic/claude-3-sonnet-20240229'"
]
},
{
"cell_type": "markdown",
"id": "d2b1ed59",
"metadata": {},
"source": [
"## 2. Define a Simple Tool"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "04b3ef5b",
"metadata": {},
"outputs": [],
"source": [
"# Weather tool implementation\n",
"def get_weather(city: str) -> dict:\n",
" \"\"\"Retrieves the current weather report for a specified city.\"\"\"\n",
" print(f'Tool: get_weather called for city: {city}')\n",
"\n",
" # Mock weather data\n",
" mock_weather_db = {\n",
" 'newyork': {\n",
" 'status': 'success',\n",
" 'report': 'The weather in New York is sunny with a temperature of 25°C.'\n",
" },\n",
" 'london': {\n",
" 'status': 'success',\n",
" 'report': \"It's cloudy in London with a temperature of 15°C.\"\n",
" },\n",
" 'tokyo': {\n",
" 'status': 'success',\n",
" 'report': 'Tokyo is experiencing light rain and a temperature of 18°C.'\n",
" },\n",
" }\n",
"\n",
" city_normalized = city.lower().replace(' ', '')\n",
"\n",
" if city_normalized in mock_weather_db:\n",
" return mock_weather_db[city_normalized]\n",
" else:\n",
" return {\n",
" 'status': 'error',\n",
" 'error_message': f\"Sorry, I don't have weather information for '{city}'.\"\n",
" }"
]
},
{
"cell_type": "markdown",
"id": "727b15c9",
"metadata": {},
"source": [
"## 3. Helper Function for Agent Interaction"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f77449bf",
"metadata": {},
"outputs": [],
"source": [
"# Agent interaction helper function\n",
"async def call_agent_async(query: str, runner, user_id, session_id):\n",
" \"\"\"Sends a query to the agent and prints the final response.\"\"\"\n",
" print(f'\\n>>> User Query: {query}')\n",
"\n",
" content = types.Content(role='user', parts=[types.Part(text=query)])\n",
" final_response_text = 'Agent did not produce a final response.'\n",
"\n",
" async for event in runner.run_async(\n",
" user_id=user_id,\n",
" session_id=session_id,\n",
" new_message=content\n",
" ):\n",
" if event.is_final_response():\n",
" if event.content and event.content.parts:\n",
" final_response_text = event.content.parts[0].text\n",
" break\n",
" print(f'<<< Agent Response: {final_response_text}')"
]
},
{
"cell_type": "markdown",
"id": "0ac87987",
"metadata": {},
"source": [
"## 4. Using Different Model Providers with ADK\n",
"\n",
"### 4.1 Using OpenAI Models"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e167d557",
"metadata": {},
"outputs": [],
"source": [
"# OpenAI model implementation\n",
"weather_agent_gpt = Agent(\n",
" name='weather_agent_gpt',\n",
" model=LiteLlm(model=MODEL_GPT_4O),\n",
" description='Provides weather information using OpenAI\\'s GPT.',\n",
" instruction=(\n",
" 'You are a helpful weather assistant powered by GPT-4o. '\n",
" \"Use the 'get_weather' tool for city weather requests. \"\n",
" 'Present information clearly.'\n",
" ),\n",
" tools=[get_weather],\n",
")\n",
"\n",
"session_service_gpt = InMemorySessionService()\n",
"session_gpt = session_service_gpt.create_session(\n",
" app_name='weather_app', user_id='user_1', session_id='session_gpt'\n",
")\n",
"\n",
"runner_gpt = Runner(\n",
" agent=weather_agent_gpt,\n",
" app_name='weather_app',\n",
" session_service=session_service_gpt,\n",
")\n",
"\n",
"async def test_gpt_agent():\n",
" print('\\n--- Testing GPT Agent ---')\n",
" await call_agent_async(\n",
" \"What's the weather in London?\",\n",
" runner=runner_gpt,\n",
" user_id='user_1',\n",
" session_id='session_gpt',\n",
" )\n",
"\n",
"# To execute in a notebook cell:\n",
"# await test_gpt_agent()"
]
},
{
"cell_type": "markdown",
"id": "f9cb0613",
"metadata": {},
"source": [
"### 4.2 Using Anthropic Models"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1c653665",
"metadata": {},
"outputs": [],
"source": [
"# Anthropic model implementation\n",
"weather_agent_claude = Agent(\n",
" name='weather_agent_claude',\n",
" model=LiteLlm(model=MODEL_CLAUDE_SONNET),\n",
" description='Provides weather information using Anthropic\\'s Claude.',\n",
" instruction=(\n",
" 'You are a helpful weather assistant powered by Claude Sonnet. '\n",
" \"Use the 'get_weather' tool for city weather requests. \"\n",
" 'Present information clearly.'\n",
" ),\n",
" tools=[get_weather],\n",
")\n",
"\n",
"session_service_claude = InMemorySessionService()\n",
"session_claude = session_service_claude.create_session(\n",
" app_name='weather_app', user_id='user_1', session_id='session_claude'\n",
")\n",
"\n",
"runner_claude = Runner(\n",
" agent=weather_agent_claude,\n",
" app_name='weather_app',\n",
" session_service=session_service_claude,\n",
")\n",
"\n",
"async def test_claude_agent():\n",
" print('\\n--- Testing Claude Agent ---')\n",
" await call_agent_async(\n",
" \"What's the weather in Tokyo?\",\n",
" runner=runner_claude,\n",
" user_id='user_1',\n",
" session_id='session_claude',\n",
" )\n",
"\n",
"# To execute in a notebook cell:\n",
"# await test_claude_agent()"
]
},
{
"cell_type": "markdown",
"id": "bf9d863b",
"metadata": {},
"source": [
"### 4.3 Using Google's Gemini Models"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "83f49d0a",
"metadata": {},
"outputs": [],
"source": [
"# Gemini model implementation\n",
"weather_agent_gemini = Agent(\n",
" name='weather_agent_gemini',\n",
" model=MODEL_GEMINI_PRO,\n",
" description='Provides weather information using Google\\'s Gemini.',\n",
" instruction=(\n",
" 'You are a helpful weather assistant powered by Gemini Pro. '\n",
" \"Use the 'get_weather' tool for city weather requests. \"\n",
" 'Present information clearly.'\n",
" ),\n",
" tools=[get_weather],\n",
")\n",
"\n",
"session_service_gemini = InMemorySessionService()\n",
"session_gemini = session_service_gemini.create_session(\n",
" app_name='weather_app', user_id='user_1', session_id='session_gemini'\n",
")\n",
"\n",
"runner_gemini = Runner(\n",
" agent=weather_agent_gemini,\n",
" app_name='weather_app',\n",
" session_service=session_service_gemini,\n",
")\n",
"\n",
"async def test_gemini_agent():\n",
" print('\\n--- Testing Gemini Agent ---')\n",
" await call_agent_async(\n",
" \"What's the weather in New York?\",\n",
" runner=runner_gemini,\n",
" user_id='user_1',\n",
" session_id='session_gemini',\n",
" )\n",
"\n",
"# To execute in a notebook cell:\n",
"# await test_gemini_agent()"
]
},
{
"cell_type": "markdown",
"id": "93bc5fd0",
"metadata": {},
"source": [
"## 5. Using LiteLLM Proxy with ADK"
]
},
{
"cell_type": "markdown",
"id": "b4275151",
"metadata": {},
"source": [
"| Variable | Description |\n",
"|----------|-------------|\n",
"| `LITELLM_PROXY_API_KEY` | The API key for the LiteLLM proxy |\n",
"| `LITELLM_PROXY_API_BASE` | The base URL for the LiteLLM proxy |\n",
"| `USE_LITELLM_PROXY` or `litellm.use_litellm_proxy` | When set to True, your request will be sent to LiteLLM proxy. |"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "256530a6",
"metadata": {},
"outputs": [],
"source": [
"# LiteLLM proxy integration\n",
"os.environ['LITELLM_PROXY_API_KEY'] = 'your-litellm-proxy-api-key'\n",
"os.environ['LITELLM_PROXY_API_BASE'] = 'your-litellm-proxy-url' # e.g., 'http://localhost:4000'\n",
"litellm.use_litellm_proxy = True\n",
"\n",
"weather_agent_proxy_env = Agent(\n",
" name='weather_agent_proxy_env',\n",
" model=LiteLlm(model='gpt-4o'),\n",
" description='Provides weather information using a model from LiteLLM proxy.',\n",
" instruction=(\n",
" 'You are a helpful weather assistant. '\n",
" \"Use the 'get_weather' tool for city weather requests. \"\n",
" 'Present information clearly.'\n",
" ),\n",
" tools=[get_weather],\n",
")\n",
"\n",
"session_service_proxy_env = InMemorySessionService()\n",
"session_proxy_env = session_service_proxy_env.create_session(\n",
" app_name='weather_app', user_id='user_1', session_id='session_proxy_env'\n",
")\n",
"\n",
"runner_proxy_env = Runner(\n",
" agent=weather_agent_proxy_env,\n",
" app_name='weather_app',\n",
" session_service=session_service_proxy_env,\n",
")\n",
"\n",
"async def test_proxy_env_agent():\n",
" print('\\n--- Testing Proxy-enabled Agent (Environment Variables) ---')\n",
" await call_agent_async(\n",
" \"What's the weather in London?\",\n",
" runner=runner_proxy_env,\n",
" user_id='user_1',\n",
" session_id='session_proxy_env',\n",
" )\n",
"\n",
"# To execute in a notebook cell:\n",
"# await test_proxy_env_agent()"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

213
cookbook/liteLLM_Baseten.ipynb vendored
View file

@ -6,19 +6,21 @@
"id": "gZx-wHJapG5w"
},
"source": [
"# Use liteLLM to call Falcon, Wizard, MPT 7B using OpenAI chatGPT Input/output\n",
"# LiteLLM with Baseten Model APIs\n",
"\n",
"* Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
"* Wizard LM: https://app.baseten.co/explore/wizardlm\n",
"* MPT 7B Base: https://app.baseten.co/explore/mpt_7b_instruct\n",
"This notebook demonstrates how to use LiteLLM with Baseten's Model APIs instead of dedicated deployments.\n",
"\n",
"\n",
"## Call all baseten llm models using OpenAI chatGPT Input/Output using liteLLM\n",
"Example call\n",
"## Example Usage\n",
"```python\n",
"model = \"q841o8w\" # baseten model version ID\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"```"
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n",
" max_tokens=1000,\n",
" temperature=0.7\n",
")\n",
"```\n",
"\n",
"## Setup"
]
},
{
@ -29,20 +31,25 @@
},
"outputs": [],
"source": [
"!pip install litellm==0.1.399\n",
"!pip install baseten urllib3"
"%pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {
"id": "VEukLhDzo4vw"
},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion"
"from litellm import completion\n",
"\n",
"# Set your Baseten API key\n",
"os.environ['BASETEN_API_KEY'] = \"\" #@param {type:\"string\"}\n",
"\n",
"# Test message\n",
"messages = [{\"role\": \"user\", \"content\": \"What is AGI?\"}]"
]
},
{
@ -51,19 +58,31 @@
"id": "4STYM2OHFNlc"
},
"source": [
"## Setup"
"## Example 1: Basic Completion\n",
"\n",
"Simple completion with the GPT-OSS 120B model"
]
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": null,
"metadata": {
"id": "DorpLxw1FHbC"
},
"outputs": [],
"source": [
"os.environ['BASETEN_API_KEY'] = \"\" #@param\n",
"messages = [{ \"content\": \"what does Baseten do? \",\"role\": \"user\"}]"
"print(\"=== Basic Completion ===\")\n",
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=messages,\n",
" max_tokens=1000,\n",
" temperature=0.7,\n",
" top_p=0.9,\n",
" presence_penalty=0.1,\n",
" frequency_penalty=0.1,\n",
")\n",
"print(f\"Response: {response.choices[0].message.content}\")\n",
"print(f\"Usage: {response.usage}\")"
]
},
{
@ -72,13 +91,14 @@
"id": "syF3dTdKFSQQ"
},
"source": [
"## Calling Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
"### Pass Your Baseten model `Version ID` as `model`"
"## Example 2: Streaming Completion\n",
"\n",
"Streaming completion with usage statistics"
]
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -86,137 +106,26 @@
"id": "rPgSoMlsojz0",
"outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': \"what does Baseten do? \\nI'm sorry, I cannot provide a specific answer as\"}}],\n",
" 'created': 1692135883.699066,\n",
" 'model': 'qvv0xeq'}"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"model = \"qvv0xeq\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7n21UroEGCGa"
},
"source": [
"## Calling Wizard LM https://app.baseten.co/explore/wizardlm\n",
"### Pass Your Baseten model `Version ID` as `model`"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "uLVWFH899lAF",
"outputId": "61c2bc74-673b-413e-bb40-179cf408523d"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': 'As an AI language model, I do not have personal beliefs or practices, but based on the information available online, Baseten is a popular name for a traditional Ethiopian dish made with injera, a spongy flatbread, and wat, a spicy stew made with meat or vegetables. It is typically served for breakfast or dinner and is a staple in Ethiopian cuisine. The name Baseten is also used to refer to a traditional Ethiopian coffee ceremony, where coffee is brewed and served in a special ceremony with music and food.'}}],\n",
" 'created': 1692135900.2806294,\n",
" 'model': 'q841o8w'}"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = \"q841o8w\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "6-TFwmPAGPXq"
},
"source": [
"## Calling mosaicml/mpt-7b https://app.baseten.co/explore/mpt_7b_instruct\n",
"### Pass Your Baseten model `Version ID` as `model`"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "gbeYZOrUE_Bp",
"outputId": "838d86ea-2143-4cb3-bc80-2acc2346c37a"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': \"\\n===================\\n\\nIt's a tool to build a local version of a game on your own machine to host\\non your website.\\n\\nIt's used to make game demos and show them on Twitter, Tumblr, and Facebook.\\n\\n\\n\\n## What's built\\n\\n- A directory of all your game directories, named with a version name and build number, with images linked to.\\n- Includes HTML to include in another site.\\n- Includes images for your icons and\"}}],\n",
" 'created': 1692135914.7472186,\n",
" 'model': '31dxrj3'}"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = \"31dxrj3\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
"print(\"=== Streaming Completion ===\")\n",
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=[{\"role\": \"user\", \"content\": \"Write a short poem about AI\"}],\n",
" stream=True,\n",
" max_tokens=500,\n",
" temperature=0.8,\n",
" stream_options={\n",
" \"include_usage\": True,\n",
" \"continuous_usage_stats\": True\n",
" },\n",
")\n",
"\n",
"print(\"Streaming response:\")\n",
"for chunk in response:\n",
" if chunk.choices and chunk.choices[0].delta.content:\n",
" print(chunk.choices[0].delta.content, end=\"\", flush=True)\n",
"print(\"\\n\")"
]
}
],
@ -234,4 +143,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
}
}

2
cookbook/litellm-ollama-docker-image/requirements.txt
View file

@ -1 +1 @@
litellm==1.55.3
litellm==1.61.15

25
cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py Normal file
View file

@ -0,0 +1,25 @@
from openai import OpenAI
client = OpenAI(
base_url="http://0.0.0.0:4000",
api_key="sk-1234",
)
BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
# Upload file
batch_input_file = client.files.create(
file=open("./bedrock_batch_completions.jsonl", "rb"),
purpose="batch",
extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
)
print(batch_input_file)
# Create batch
batch = client.batches.create(
input_file_id=batch_input_file.id,
endpoint="/v1/chat/completions",
completion_window="24h",
metadata={"description": "Test batch job"},
)
print(batch)

128
cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl Normal file
View file

@ -0,0 +1,128 @@
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}

279
cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md Normal file
View file

@ -0,0 +1,279 @@
# Braintrust Prompt Wrapper for LiteLLM
This directory contains a wrapper server that enables LiteLLM to use prompts from [Braintrust](https://www.braintrust.dev/) through the generic prompt management API.
## Architecture
```
┌─────────────┐ ┌──────────────────────┐ ┌─────────────┐
│ LiteLLM │ ──────> │ Wrapper Server │ ──────> │ Braintrust │
│ Client │ │ (This Server) │ │ API │
└─────────────┘ └──────────────────────┘ └─────────────┘
Uses generic Transforms Stores actual
prompt manager Braintrust format prompt templates
to LiteLLM format
```
## Components
### 1. Generic Prompt Manager (`litellm/integrations/generic_prompt_management/`)
A generic client that can work with any API implementing the `/beta/litellm_prompt_management` endpoint.
**Expected API Response Format:**
```json
{
"prompt_id": "string",
"prompt_template": [
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Hello {name}"}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 100
}
}
```
### 2. Braintrust Wrapper Server (`braintrust_prompt_wrapper_server.py`)
A FastAPI server that:
- Implements the `/beta/litellm_prompt_management` endpoint
- Fetches prompts from Braintrust API
- Transforms Braintrust response format to LiteLLM format
## Setup
### Install Dependencies
```bash
pip install fastapi uvicorn httpx litellm
```
### Set Environment Variables
```bash
export BRAINTRUST_API_KEY="your-braintrust-api-key"
```
## Usage
### Step 1: Start the Wrapper Server
```bash
python braintrust_prompt_wrapper_server.py
```
The server will start on `http://localhost:8080` by default.
You can customize the port and host:
```bash
export PORT=8000
export HOST=0.0.0.0
python braintrust_prompt_wrapper_server.py
```
### Step 2: Use with LiteLLM
```python
import litellm
from litellm.integrations.generic_prompt_management import GenericPromptManager
# Configure the generic prompt manager to use your wrapper server
generic_config = {
"api_base": "http://localhost:8080",
"api_key": "your-braintrust-api-key", # Will be passed to Braintrust
"timeout": 30,
}
# Create the prompt manager
prompt_manager = GenericPromptManager(**generic_config)
# Use with completion
response = litellm.completion(
model="generic_prompt/gpt-4",
prompt_id="your-braintrust-prompt-id",
prompt_variables={"name": "World"}, # Variables to substitute
messages=[{"role": "user", "content": "Additional message"}]
)
print(response)
```
### Step 3: Direct API Testing
You can also test the wrapper API directly:
```bash
# Test with curl
curl -H "Authorization: Bearer YOUR_BRAINTRUST_TOKEN" \
"http://localhost:8080/beta/litellm_prompt_management?prompt_id=YOUR_PROMPT_ID"
# Health check
curl http://localhost:8080/health
# Service info
curl http://localhost:8080/
```
## API Documentation
Once the server is running, visit:
- Swagger UI: `http://localhost:8080/docs`
- ReDoc: `http://localhost:8080/redoc`
## Braintrust Format Transformation
The wrapper automatically transforms Braintrust's response format:
**Braintrust API Response:**
```json
{
"id": "prompt-123",
"prompt_data": {
"prompt": {
"type": "chat",
"messages": [
{
"role": "system",
"content": "You are a helpful assistant"
}
]
},
"options": {
"model": "gpt-4",
"params": {
"temperature": 0.7,
"max_tokens": 100
}
}
}
}
```
**Transformed to LiteLLM Format:**
```json
{
"prompt_id": "prompt-123",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 100
}
}
```
## Supported Parameters
The wrapper automatically maps these Braintrust parameters to LiteLLM:
- `temperature`
- `max_tokens` / `max_completion_tokens`
- `top_p`
- `frequency_penalty`
- `presence_penalty`
- `n`
- `stop`
- `response_format`
- `tool_choice`
- `function_call`
- `tools`
## Variable Substitution
The generic prompt manager supports simple variable substitution:
```python
# In your Braintrust prompt:
# "Hello {name}, welcome to {place}!"
# In your code:
prompt_variables = {
"name": "Alice",
"place": "Wonderland"
}
# Result:
# "Hello Alice, welcome to Wonderland!"
```
Supports both `{variable}` and `{{variable}}` syntax.
## Error Handling
The wrapper provides detailed error messages:
- **401**: Missing or invalid Braintrust API token
- **404**: Prompt not found in Braintrust
- **502**: Failed to connect to Braintrust API
- **500**: Error transforming response
## Production Deployment
For production use:
1. **Use HTTPS**: Deploy behind a reverse proxy with SSL
2. **Authentication**: Add authentication to the wrapper endpoint if needed
3. **Rate Limiting**: Implement rate limiting to prevent abuse
4. **Caching**: Consider caching prompt responses
5. **Monitoring**: Add logging and monitoring
Example with Docker:
```dockerfile
FROM python:3.11-slim
WORKDIR /app
RUN pip install fastapi uvicorn httpx
COPY braintrust_prompt_wrapper_server.py .
ENV PORT=8080
ENV HOST=0.0.0.0
EXPOSE 8080
CMD ["python", "braintrust_prompt_wrapper_server.py"]
```
## Extending to Other Providers
This pattern can be used with any prompt management provider:
1. Create a wrapper server that implements `/beta/litellm_prompt_management`
2. Transform the provider's response to LiteLLM format
3. Use the generic prompt manager to connect
Example providers:
- Langsmith
- PromptLayer
- Humanloop
- Custom internal systems
## Troubleshooting
### "No Braintrust API token provided"
- Set `BRAINTRUST_API_KEY` environment variable
- Or pass token in `Authorization: Bearer TOKEN` header
### "Failed to connect to Braintrust API"
- Check your internet connection
- Verify Braintrust API is accessible
- Check firewall settings
### "Prompt not found"
- Verify the prompt ID exists in Braintrust
- Check that your API token has access to the prompt
## License
This wrapper is part of the LiteLLM project and follows the same license.

274
cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py Normal file
View file

@ -0,0 +1,274 @@
"""
Mock server that implements the /beta/litellm_prompt_management endpoint
and acts as a wrapper for calling the Braintrust API.
This server transforms Braintrust's prompt API response into the format
expected by LiteLLM's generic prompt management client.
Usage:
python braintrust_prompt_wrapper_server.py
# Then test with:
curl -H "Authorization: Bearer YOUR_BRAINTRUST_TOKEN" \
"http://localhost:8080/beta/litellm_prompt_management?prompt_id=YOUR_PROMPT_ID"
"""
import json
import os
from typing import Any, Dict, List, Optional
import httpx
from fastapi import FastAPI, HTTPException, Header, Query
from fastapi.responses import JSONResponse
import uvicorn
app = FastAPI(
title="Braintrust Prompt Wrapper",
description="Wrapper server for Braintrust prompts to work with LiteLLM",
version="1.0.0",
)
def transform_braintrust_message(message: Dict[str, Any]) -> Dict[str, str]:
"""
Transform a Braintrust message to LiteLLM format.
Braintrust message format:
{
"role": "system",
"content": "...",
"name": "..." (optional)
}
LiteLLM format:
{
"role": "system",
"content": "..."
}
"""
result = {
"role": message.get("role", "user"),
"content": message.get("content", ""),
}
# Include name if present
if "name" in message:
result["name"] = message["name"]
return result
def transform_braintrust_response(
braintrust_response: Dict[str, Any],
) -> Dict[str, Any]:
"""
Transform Braintrust API response to LiteLLM prompt management format.
Braintrust response format:
{
"objects": [{
"id": "prompt_id",
"prompt_data": {
"prompt": {
"type": "chat",
"messages": [...],
"tools": "..."
},
"options": {
"model": "gpt-4",
"params": {
"temperature": 0.7,
"max_tokens": 100,
...
}
}
}
}]
}
LiteLLM format:
{
"prompt_id": "prompt_id",
"prompt_template": [...],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {...}
}
"""
# Extract the first object from the objects array if it exists
if "objects" in braintrust_response and len(braintrust_response["objects"]) > 0:
prompt_object = braintrust_response["objects"][0]
else:
prompt_object = braintrust_response
prompt_data = prompt_object.get("prompt_data", {})
prompt_info = prompt_data.get("prompt", {})
options = prompt_data.get("options", {})
# Extract messages
messages = prompt_info.get("messages", [])
transformed_messages = [transform_braintrust_message(msg) for msg in messages]
# Extract model
model = options.get("model")
# Extract optional parameters
params = options.get("params", {})
optional_params: Dict[str, Any] = {}
# Map common parameters
param_mapping = {
"temperature": "temperature",
"max_tokens": "max_tokens",
"max_completion_tokens": "max_tokens", # Alternative name
"top_p": "top_p",
"frequency_penalty": "frequency_penalty",
"presence_penalty": "presence_penalty",
"n": "n",
"stop": "stop",
}
for braintrust_param, litellm_param in param_mapping.items():
if braintrust_param in params:
value = params[braintrust_param]
if value is not None:
optional_params[litellm_param] = value
# Handle response_format
if "response_format" in params:
optional_params["response_format"] = params["response_format"]
# Handle tool_choice
if "tool_choice" in params:
optional_params["tool_choice"] = params["tool_choice"]
# Handle function_call
if "function_call" in params:
optional_params["function_call"] = params["function_call"]
# Add tools if present
if "tools" in prompt_info and prompt_info["tools"]:
optional_params["tools"] = prompt_info["tools"]
# Handle tool_functions from prompt_data
if "tool_functions" in prompt_data and prompt_data["tool_functions"]:
optional_params["tool_functions"] = prompt_data["tool_functions"]
return {
"prompt_id": prompt_object.get("id"),
"prompt_template": transformed_messages,
"prompt_template_model": model,
"prompt_template_optional_params": optional_params if optional_params else None,
}
@app.get("/beta/litellm_prompt_management")
async def get_prompt(
prompt_id: str = Query(..., description="The Braintrust prompt ID to fetch"),
authorization: Optional[str] = Header(
None, description="Bearer token for Braintrust API"
),
) -> JSONResponse:
"""
Fetch a prompt from Braintrust and transform it to LiteLLM format.
Args:
prompt_id: The Braintrust prompt ID
authorization: Bearer token for Braintrust API (from header)
Returns:
JSONResponse with the transformed prompt data
"""
# Extract token from Authorization header or environment
braintrust_token = None
if authorization and authorization.startswith("Bearer "):
braintrust_token = authorization.replace("Bearer ", "")
else:
braintrust_token = os.getenv("BRAINTRUST_API_KEY")
if not braintrust_token:
raise HTTPException(
status_code=401,
detail="No Braintrust API token provided. Pass via Authorization header or set BRAINTRUST_API_KEY environment variable.",
)
# Call Braintrust API
braintrust_url = f"https://api.braintrust.dev/v1/prompt/{prompt_id}"
headers = {
"Authorization": f"Bearer {braintrust_token}",
"Accept": "application/json",
}
print(f"headers: {headers}")
print(f"braintrust_url: {braintrust_url}")
print(f"braintrust_token: {braintrust_token}")
try:
async with httpx.AsyncClient(timeout=30.0) as client:
response = await client.get(braintrust_url, headers=headers)
response.raise_for_status()
braintrust_data = response.json()
except httpx.HTTPStatusError as e:
raise HTTPException(
status_code=e.response.status_code,
detail=f"Braintrust API error: {e.response.text}",
)
except httpx.RequestError as e:
raise HTTPException(
status_code=502,
detail=f"Failed to connect to Braintrust API: {str(e)}",
)
except json.JSONDecodeError as e:
raise HTTPException(
status_code=502,
detail=f"Failed to parse Braintrust API response: {str(e)}",
)
print(f"braintrust_data: {braintrust_data}")
# Transform the response
try:
transformed_data = transform_braintrust_response(braintrust_data)
print(f"transformed_data: {transformed_data}")
return JSONResponse(content=transformed_data)
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to transform Braintrust response: {str(e)}",
)
@app.get("/health")
async def health_check():
"""Health check endpoint."""
return {"status": "healthy", "service": "braintrust-prompt-wrapper"}
@app.get("/")
async def root():
"""Root endpoint with service information."""
return {
"service": "Braintrust Prompt Wrapper for LiteLLM",
"version": "1.0.0",
"endpoints": {
"prompt_management": "/beta/litellm_prompt_management?prompt_id=<id>",
"health": "/health",
},
"documentation": "/docs",
}
def main():
"""Run the server."""
port = int(os.getenv("PORT", "8080"))
host = os.getenv("HOST", "0.0.0.0")
print(f"🚀 Starting Braintrust Prompt Wrapper Server on {host}:{port}")
print(f"📚 API Documentation available at http://{host}:{port}/docs")
print(
f"🔑 Make sure to set BRAINTRUST_API_KEY environment variable or pass token in Authorization header"
)
uvicorn.run(app, host=host, port=port)
if __name__ == "__main__":
main()

62
cookbook/litellm_proxy_server/cli_token_usage.py Normal file
View file

@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""
Example: Using CLI token with LiteLLM SDK
This example shows how to use the CLI authentication token
in your Python scripts after running `litellm-proxy login`.
"""
from textwrap import indent
import litellm
LITELLM_BASE_URL = "http://localhost:4000/"
def main():
"""Using CLI token with LiteLLM SDK"""
print("🚀 Using CLI Token with LiteLLM SDK")
print("=" * 40)
#litellm._turn_on_debug()
# Get the CLI token
api_key = litellm.get_litellm_gateway_api_key()
if not api_key:
print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
return
print("✅ Found CLI token.")
available_models = litellm.get_valid_models(
check_provider_endpoint=True,
custom_llm_provider="litellm_proxy",
api_key=api_key,
api_base=LITELLM_BASE_URL
)
print("✅ Available models:")
if available_models:
for i, model in enumerate(available_models, 1):
print(f" {i:2d}. {model}")
else:
print(" No models available")
# Use with LiteLLM
try:
response = litellm.completion(
model="litellm_proxy/gemini/gemini-2.5-flash",
messages=[{"role": "user", "content": "Hello from CLI token!"}],
api_key=api_key,
base_url=LITELLM_BASE_URL
)
print(f"✅ LLM Response: {response.model_dump_json(indent=4)}")
except Exception as e:
print(f"❌ Error: {e}")
if __name__ == "__main__":
main()
print("\n💡 Tips:")
print("1. Run 'litellm-proxy login' to authenticate first")
print("2. Replace 'https://your-proxy.com' with your actual proxy URL")
print("3. The token is stored locally at ~/.litellm/token.json")

46
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_1/grafana_dashboard.json
View file

@ -32,7 +32,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
@ -110,7 +110,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum(rate(litellm_self_latency_bucket{self=\"self\"}[1m])) by (le))",
@ -125,7 +125,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
@ -216,7 +216,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(increase(litellm_spend_metric_total[30d])) by (hashed_api_key)",
@ -232,7 +232,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
@ -309,7 +309,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum by (model) (increase(litellm_requests_metric_total[5m]))",
@ -324,7 +324,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
@ -375,7 +375,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(increase(litellm_llm_api_failed_requests_metric_total[1h]))",
@ -390,7 +390,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
@ -468,7 +468,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(increase(litellm_spend_metric_total[30d])) by (model)",
@ -483,7 +483,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
@ -560,7 +560,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(increase(litellm_total_tokens_total[5m])) by (model)",
@ -579,7 +579,27 @@
"style": "dark",
"tags": [],
"templating": {
"list": []
"list": [
{
"current": {
"selected": false,
"text": "prometheus",
"value": "edx8memhpd9tsa"
},
"hide": 0,
"includeAll": false,
"label": "datasource",
"multi": false,
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-1h",

52
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_v2/grafana_dashboard.json
View file

@ -37,7 +37,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"description": "Total requests per second made to proxy - success + failure ",
"fieldConfig": {
@ -119,7 +119,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"disableTextWrap": false,
"editorMode": "code",
@ -138,7 +138,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"description": "Failures per second by Exception Class",
"fieldConfig": {
@ -220,7 +220,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"disableTextWrap": false,
"editorMode": "code",
@ -239,7 +239,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"description": "Average Response latency (seconds)",
"fieldConfig": {
@ -346,7 +346,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"disableTextWrap": false,
"editorMode": "code",
@ -361,7 +361,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, sum(rate(litellm_request_total_latency_metric_bucket[2m])) by (le))",
@ -391,7 +391,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"description": "x-ratelimit-remaining-requests returning from LLM APIs",
"fieldConfig": {
@ -473,7 +473,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "topk(5, sort(litellm_remaining_requests))",
@ -488,7 +488,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"description": "x-ratelimit-remaining-tokens from LLM API ",
"fieldConfig": {
@ -570,7 +570,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "topk(5, sort(litellm_remaining_tokens))",
@ -598,7 +598,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"description": "Requests per second by Key Alias (keys are LiteLLM Virtual Keys). If key is None - means no Alias Set ",
"fieldConfig": {
@ -679,7 +679,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(litellm_proxy_total_requests_metric_total[2m])) by (api_key_alias)\n",
@ -694,7 +694,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"description": "Requests per second by Team Alias. If team is None - means no team alias Set ",
"fieldConfig": {
@ -775,7 +775,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "bdiyc60dco54we"
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "sum(rate(litellm_proxy_total_requests_metric_total[2m])) by (team_alias)\n",
@ -792,7 +792,27 @@
"schemaVersion": 40,
"tags": [],
"templating": {
"list": []
"list": [
{
"current": {
"selected": false,
"text": "prometheus",
"value": "edx8memhpd9tsb"
},
"hide": 0,
"includeAll": false,
"label": "datasource",
"multi": false,
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",

36
cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py Normal file
View file

@ -0,0 +1,36 @@
"""
Use LiteLLM Proxy MCP Gateway to call MCP tools.
When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
"""
import openai
client = openai.OpenAI(
api_key="sk-1234", # paste your litellm proxy api key here
base_url="http://localhost:4000" # paste your litellm proxy base url here
)
print("Making API request to Responses API with MCP tools")
response = client.responses.create(
model="gpt-5",
input=[
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
}
],
tools=[
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never"
}
],
stream=True,
tool_choice="required"
)
for chunk in response:
print("response chunk: ", chunk)

20
cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml Normal file
View file

@ -0,0 +1,20 @@
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
key_management_system: "custom"
key_management_settings:
custom_secret_manager: my_secret_manager.InMemorySecretManager
store_virtual_keys: true
prefix_for_stored_virtual_keys: "litellm/"
access_mode: "read_and_write"
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY # Read from custom secret manager
- model_name: claude-3-5-sonnet
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY # Read from custom secret manager

79
cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py Normal file
View file

@ -0,0 +1,79 @@
"""
Example custom secret manager for LiteLLM Proxy.
This is a simple in-memory secret manager for testing purposes.
In production, replace this with your actual secret management system.
"""
from typing import Optional, Union
import httpx
from litellm.integrations.custom_secret_manager import CustomSecretManager
class InMemorySecretManager(CustomSecretManager):
def __init__(self):
super().__init__(secret_manager_name="in_memory_secrets")
# Store your secrets in memory
print("INITIALIZING CUSTOM SECRET MANAGER IN MEMORY")
self.secrets = {}
print("CUSTOM SECRET MANAGER IN MEMORY INITIALIZED")
async def async_read_secret(
self,
secret_name: str,
optional_params: Optional[dict] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Optional[str]:
"""Read secret asynchronously"""
print("READING SECRET ASYNCHRONOUSLY")
print("SECRET NAME: %s", secret_name)
print("SECRET: %s", self.secrets.get(secret_name))
return self.secrets.get(secret_name)
def sync_read_secret(
self,
secret_name: str,
optional_params: Optional[dict] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Optional[str]:
"""Read secret synchronously"""
from litellm._logging import verbose_proxy_logger
verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}")
value = self.secrets.get(secret_name)
verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: READ SECRET: {value}")
return value
async def async_write_secret(
self,
secret_name: str,
secret_value: str,
description: Optional[str] = None,
optional_params: Optional[dict] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
tags: Optional[Union[dict, list]] = None,
) -> dict:
"""Write a secret to the in-memory store"""
self.secrets[secret_name] = secret_value
print("ALL SECRETS=%s", self.secrets)
return {
"status": "success",
"secret_name": secret_name,
"description": description,
}
async def async_delete_secret(
self,
secret_name: str,
recovery_window_in_days: Optional[int] = 7,
optional_params: Optional[dict] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> dict:
"""Delete a secret from the in-memory store"""
if secret_name in self.secrets:
del self.secrets[secret_name]
return {"status": "deleted", "secret_name": secret_name}
return {"status": "not_found", "secret_name": secret_name}

2
cookbook/litellm_router_load_test/memory_usage/router_endpoint.py
View file

@ -5,7 +5,7 @@ import os
import litellm
from litellm import Router
from dotenv import load_dotenv
import uuid
from litellm._uuid import uuid
load_dotenv()

2
cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py
View file

@ -12,7 +12,7 @@ sys.path.insert(
import litellm
from litellm import Router
from dotenv import load_dotenv
import uuid
from litellm._uuid import uuid
load_dotenv()

2
cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py
View file

@ -12,7 +12,7 @@ sys.path.insert(
import litellm
from litellm import Router
from dotenv import load_dotenv
import uuid
from litellm._uuid import uuid
load_dotenv()

114
cookbook/livekit_agent_sdk/README.md Normal file
View file

@ -0,0 +1,114 @@
# LiveKit Voice Agent with LiteLLM Gateway
Simple example showing how to use LiveKit's xAI realtime plugin with LiteLLM as a proxy. This lets you switch between xAI, OpenAI, and Azure realtime APIs without changing your code.
## Quick Start
### 1. Install dependencies
```bash
pip install livekit-agents[xai] websockets
```
### 2. Start LiteLLM proxy
```bash
# With xAI
export XAI_API_KEY="your-xai-key"
litellm --config config.yaml --port 4000
```
### 3. Run the voice agent
```bash
python main.py
```
Type your message and get a voice response from Grok!
## Configuration
Set these environment variables if needed:
```bash
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
export LITELLM_MODEL="grok-voice-agent"
```
Or use the defaults - connects to `http://localhost:4000` by default.
## Example Config File
Create a `config.yaml` with your realtime models:
```yaml
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-2-vision-1212
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
- model_name: openai-voice-agent
litellm_params:
model: gpt-4o-realtime-preview
api_key: os.environ/OPENAI_API_KEY
model_info:
mode: realtime
general_settings:
master_key: sk-1234
```
Then start: `litellm --config config.yaml --port 4000`
## How It Works
LiveKit's xAI plugin connects through LiteLLM proxy by setting `base_url`:
```python
from livekit.plugins import xai
model = xai.realtime.RealtimeModel(
voice="ara",
api_key="sk-1234", # LiteLLM proxy key
base_url="http://localhost:4000", # Point to LiteLLM
)
```
## Switching Providers
Just change the model in your config - no code changes needed:
**xAI Grok:**
```yaml
model: xai/grok-2-vision-1212
```
**OpenAI:**
```yaml
model: gpt-4o-realtime-preview
```
**Azure OpenAI:**
```yaml
model: azure/gpt-4o-realtime-preview
api_base: https://your-endpoint.openai.azure.com/
```
## Why Use LiteLLM?
- ✅ **Switch providers** without changing agent code
- ✅ **Cost tracking** across all voice sessions
- ✅ **Rate limiting** and budgets
- ✅ **Load balancing** across multiple API keys
- ✅ **Fallbacks** to backup models
## Learn More
- [LiveKit xAI Realtime Tutorial](/docs/tutorials/livekit_xai_realtime)
- [xAI Realtime Docs](/docs/providers/xai_realtime)
- [LiveKit Agents Documentation](https://docs.livekit.io/agents/)
- [LiteLLM Realtime API](/docs/realtime)

21
cookbook/livekit_agent_sdk/config.example.yaml Normal file
View file

@ -0,0 +1,21 @@
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-2-vision-1212
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
- model_name: openai-voice-agent
litellm_params:
model: gpt-4o-realtime-preview
api_key: os.environ/OPENAI_API_KEY
model_info:
mode: realtime
litellm_settings:
drop_params: True
telemetry: False
general_settings:
master_key: sk-1234 # Change this to a secure key

112
cookbook/livekit_agent_sdk/main.py Normal file
View file

@ -0,0 +1,112 @@
"""
Simple xAI Voice Agent using LiveKit SDK with LiteLLM Gateway
This example shows how to use LiveKit's xAI realtime plugin through LiteLLM proxy.
LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI,
and Azure realtime APIs without changing your agent code.
"""
import asyncio
import json
import os
import websockets
# Configuration
PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
MODEL = os.getenv("LITELLM_MODEL", "grok-voice-agent")
async def run_voice_agent():
"""
Simple voice agent that:
1. Connects to xAI realtime API through LiteLLM proxy
2. Sends a user message
3. Streams back the response
"""
url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}"
headers = {"Authorization": f"Bearer {API_KEY}"}
print(f"🎙️ Connecting to voice agent...")
print(f" Model: {MODEL}")
print(f" Proxy: {PROXY_URL}")
print()
async with websockets.connect(url, additional_headers=headers) as ws:
# Receive initial connection event
initial = json.loads(await ws.recv())
print(f"✅ Connected! Event: {initial['type']}\n")
# Get user input
user_message = input("💬 Your message: ").strip()
if not user_message:
user_message = "Tell me a fun fact about AI!"
print(f"\n🤖 Sending to {MODEL}...\n")
# Send user message
await ws.send(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": user_message}]
}
}))
# Request response
await ws.send(json.dumps({
"type": "response.create",
"response": {"modalities": ["text", "audio"]}
}))
# Stream response
print("🎤 Response: ", end='', flush=True)
transcript = []
try:
while True:
msg = await asyncio.wait_for(ws.recv(), timeout=15.0)
event = json.loads(msg)
# Capture transcript deltas
if event['type'] == 'response.output_audio_transcript.delta':
delta = event.get('delta', '')
if delta:
print(delta, end='', flush=True)
transcript.append(delta)
# Done when response completes
elif event['type'] == 'response.done':
break
except asyncio.TimeoutError:
pass
print("\n")
if transcript:
print(f"✅ Complete response: {''.join(transcript)}")
await ws.close()
def main():
"""Run the voice agent"""
print("=" * 70)
print("LiveKit xAI Voice Agent via LiteLLM Proxy")
print("=" * 70)
print()
try:
asyncio.run(run_voice_agent())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
except Exception as e:
print(f"\n❌ Error: {e}")
print("\nMake sure LiteLLM proxy is running:")
print(f" litellm --config config.yaml --port 4000")
if __name__ == "__main__":
main()

2
cookbook/livekit_agent_sdk/requirements.txt Normal file
View file

@ -0,0 +1,2 @@
livekit-agents[xai]>=1.3.12
websockets>=15.0.1

172
cookbook/logging_observability/LiteLLM_Arize.ipynb vendored Normal file
View file

@ -0,0 +1,172 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "4FbDOmcj2VkM"
},
"source": [
"## Use LiteLLM with Arize\n",
"https://docs.litellm.ai/docs/observability/arize_integration\n",
"\n",
"This method uses the litellm proxy to send the data to Arize. The callback is set in the litellm config below, instead of using OpenInference tracing."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "21W8Woog26Ns"
},
"source": [
"## Install Dependencies"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "xrjKLBxhxu2L"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: litellm in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (1.54.1)\n",
"Requirement already satisfied: aiohttp in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (3.11.10)\n",
"Requirement already satisfied: click in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (8.1.7)\n",
"Requirement already satisfied: httpx<0.28.0,>=0.23.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.27.2)\n",
"Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (8.5.0)\n",
"Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (3.1.4)\n",
"Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (4.23.0)\n",
"Requirement already satisfied: openai>=1.55.3 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (1.57.1)\n",
"Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (2.10.3)\n",
"Requirement already satisfied: python-dotenv>=0.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (1.0.1)\n",
"Requirement already satisfied: requests<3.0.0,>=2.31.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (2.32.3)\n",
"Requirement already satisfied: tiktoken>=0.7.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.7.0)\n",
"Requirement already satisfied: tokenizers in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.21.0)\n",
"Requirement already satisfied: anyio in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (4.7.0)\n",
"Requirement already satisfied: certifi in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (2024.8.30)\n",
"Requirement already satisfied: httpcore==1.* in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (1.0.7)\n",
"Requirement already satisfied: idna in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (3.10)\n",
"Requirement already satisfied: sniffio in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (1.3.1)\n",
"Requirement already satisfied: h11<0.15,>=0.13 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpcore==1.*->httpx<0.28.0,>=0.23.0->litellm) (0.14.0)\n",
"Requirement already satisfied: zipp>=3.20 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from importlib-metadata>=6.8.0->litellm) (3.21.0)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jinja2<4.0.0,>=3.1.2->litellm) (3.0.2)\n",
"Requirement already satisfied: attrs>=22.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (24.2.0)\n",
"Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2024.10.1)\n",
"Requirement already satisfied: referencing>=0.28.4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.35.1)\n",
"Requirement already satisfied: rpds-py>=0.7.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.22.3)\n",
"Requirement already satisfied: distro<2,>=1.7.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (1.9.0)\n",
"Requirement already satisfied: jiter<1,>=0.4.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (0.6.1)\n",
"Requirement already satisfied: tqdm>4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (4.67.1)\n",
"Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (4.12.2)\n",
"Requirement already satisfied: annotated-types>=0.6.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->litellm) (0.7.0)\n",
"Requirement already satisfied: pydantic-core==2.27.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->litellm) (2.27.1)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from requests<3.0.0,>=2.31.0->litellm) (3.4.0)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from requests<3.0.0,>=2.31.0->litellm) (2.0.7)\n",
"Requirement already satisfied: regex>=2022.1.18 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from tiktoken>=0.7.0->litellm) (2024.11.6)\n",
"Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (2.4.4)\n",
"Requirement already satisfied: aiosignal>=1.1.2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.3.1)\n",
"Requirement already satisfied: frozenlist>=1.1.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.5.0)\n",
"Requirement already satisfied: multidict<7.0,>=4.5 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (6.1.0)\n",
"Requirement already satisfied: propcache>=0.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (0.2.1)\n",
"Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.18.3)\n",
"Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from tokenizers->litellm) (0.26.5)\n",
"Requirement already satisfied: filelock in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (3.16.1)\n",
"Requirement already satisfied: fsspec>=2023.5.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (2024.10.0)\n",
"Requirement already satisfied: packaging>=20.9 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (24.2)\n",
"Requirement already satisfied: pyyaml>=5.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (6.0.2)\n"
]
}
],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "jHEu-TjZ29PJ"
},
"source": [
"## Set Env Variables"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"id": "QWd9rTysxsWO"
},
"outputs": [],
"source": [
"import litellm\n",
"import os\n",
"from getpass import getpass\n",
"\n",
"os.environ[\"ARIZE_SPACE_KEY\"] = getpass(\"Enter your Arize space key: \")\n",
"os.environ[\"ARIZE_API_KEY\"] = getpass(\"Enter your Arize API key: \")\n",
"os.environ['OPENAI_API_KEY']= getpass(\"Enter your OpenAI API key: \")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's run a completion call and see the traces in Arize"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Hello! Nice to meet you, OpenAI. How can I assist you today?\n"
]
}
],
"source": [
"# set arize as a callback, litellm will send the data to arize\n",
"litellm.callbacks = [\"arize\"]\n",
" \n",
"# openai call\n",
"response = litellm.completion(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages=[\n",
" {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n",
" ]\n",
")\n",
"print(response.choices[0].message.content)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

252
cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb vendored Normal file
View file

@ -0,0 +1,252 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## LLM Ops Stack - LiteLLM Proxy + Langfuse \n",
"\n",
"This notebook demonstrates how to use LiteLLM Proxy with Langfuse \n",
"- Use LiteLLM Proxy for calling 100+ LLMs in OpenAI format\n",
"- Use Langfuse for viewing request / response traces \n",
"\n",
"\n",
"In this notebook we will setup LiteLLM Proxy to make requests to OpenAI, Anthropic, Bedrock and automatically log traces to Langfuse."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1. Setup LiteLLM Proxy\n",
"\n",
"### 1.1 Define .env variables \n",
"Define .env variables on the container that litellm proxy is running on.\n",
"```bash\n",
"## LLM API Keys\n",
"OPENAI_API_KEY=sk-proj-1234567890\n",
"ANTHROPIC_API_KEY=sk-ant-api03-1234567890\n",
"AWS_ACCESS_KEY_ID=1234567890\n",
"AWS_SECRET_ACCESS_KEY=1234567890\n",
"\n",
"## Langfuse Logging \n",
"LANGFUSE_PUBLIC_KEY=\"pk-lf-xxxx9\"\n",
"LANGFUSE_SECRET_KEY=\"sk-lf-xxxx9\"\n",
"LANGFUSE_HOST=\"https://us.cloud.langfuse.com\"\n",
"```\n",
"\n",
"\n",
"### 1.1 Setup LiteLLM Proxy Config yaml \n",
"```yaml\n",
"model_list:\n",
" - model_name: gpt-4o\n",
" litellm_params:\n",
" model: openai/gpt-4o\n",
" api_key: os.environ/OPENAI_API_KEY\n",
" - model_name: claude-3-5-sonnet-20241022\n",
" litellm_params:\n",
" model: anthropic/claude-3-5-sonnet-20241022\n",
" api_key: os.environ/ANTHROPIC_API_KEY\n",
" - model_name: us.amazon.nova-micro-v1:0\n",
" litellm_params:\n",
" model: bedrock/us.amazon.nova-micro-v1:0\n",
" aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID\n",
" aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY\n",
"\n",
"litellm_settings:\n",
" callbacks: [\"langfuse\"]\n",
"\n",
"\n",
"```"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2. Make LLM Requests to LiteLLM Proxy\n",
"\n",
"Now we will make our first LLM request to LiteLLM Proxy"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 2.1 Setup Client Side Variables to point to LiteLLM Proxy\n",
"Set `LITELLM_PROXY_BASE_URL` to the base url of the LiteLLM Proxy and `LITELLM_VIRTUAL_KEY` to the virtual key you want to use for Authentication to LiteLLM Proxy. (Note: In this initial setup you can)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"\n",
"LITELLM_PROXY_BASE_URL=\"http://0.0.0.0:4000\"\n",
"LITELLM_VIRTUAL_KEY=\"sk-oXXRa1xxxxxxxxxxx\""
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"ChatCompletion(id='chatcmpl-B0sq6QkOKNMJ0dwP3x7OoMqk1jZcI', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='Langfuse is a platform designed to monitor, observe, and troubleshoot AI and large language model (LLM) applications. It provides features that help developers gain insights into how their AI systems are performing, make debugging easier, and optimize the deployment of models. Langfuse allows for tracking of model interactions, collecting telemetry, and visualizing data, which is crucial for understanding the behavior of AI models in production environments. This kind of tool is particularly useful for developers working with language models who need to ensure reliability and efficiency in their applications.', refusal=None, role='assistant', audio=None, function_call=None, tool_calls=None))], created=1739550502, model='gpt-4o-2024-08-06', object='chat.completion', service_tier='default', system_fingerprint='fp_523b9b6e5f', usage=CompletionUsage(completion_tokens=109, prompt_tokens=13, total_tokens=122, completion_tokens_details=CompletionTokensDetails(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0), prompt_tokens_details=PromptTokensDetails(audio_tokens=0, cached_tokens=0)))"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import openai\n",
"client = openai.OpenAI(\n",
" api_key=LITELLM_VIRTUAL_KEY,\n",
" base_url=LITELLM_PROXY_BASE_URL\n",
")\n",
"\n",
"response = client.chat.completions.create(\n",
" model=\"gpt-4o\",\n",
" messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"what is Langfuse?\"\n",
" }\n",
" ],\n",
")\n",
"\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 2.3 View Traces on Langfuse\n",
"LiteLLM will send the request / response, model, tokens (input + output), cost to Langfuse.\n",
"\n",
"![image_description](litellm_proxy_langfuse.png)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 2.4 Call Anthropic, Bedrock models \n",
"\n",
"Now we can call `us.amazon.nova-micro-v1:0` and `claude-3-5-sonnet-20241022` models defined on your config.yaml both in the OpenAI request / response format."
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"ChatCompletion(id='chatcmpl-7756e509-e61f-4f5e-b5ae-b7a41013522a', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content=\"Langfuse is an observability tool designed specifically for machine learning models and applications built with natural language processing (NLP) and large language models (LLMs). It focuses on providing detailed insights into how these models perform in real-world scenarios. Here are some key features and purposes of Langfuse:\\n\\n1. **Real-time Monitoring**: Langfuse allows developers to monitor the performance of their NLP and LLM applications in real time. This includes tracking the inputs and outputs of the models, as well as any errors or issues that arise during operation.\\n\\n2. **Error Tracking**: It helps in identifying and tracking errors in the models' outputs. By analyzing incorrect or unexpected responses, developers can pinpoint where and why errors occur, facilitating more effective debugging and improvement.\\n\\n3. **Performance Metrics**: Langfuse provides various performance metrics, such as latency, throughput, and error rates. These metrics help developers understand how well their models are performing under different conditions and workloads.\\n\\n4. **Traceability**: It offers detailed traceability of requests and responses, allowing developers to follow the path of a request through the system and see how it is processed by the model at each step.\\n\\n5. **User Feedback Integration**: Langfuse can integrate user feedback to provide context for model outputs. This helps in understanding how real users are interacting with the model and how its outputs align with user expectations.\\n\\n6. **Customizable Dashboards**: Users can create custom dashboards to visualize the data collected by Langfuse. These dashboards can be tailored to highlight the most important metrics and insights for a specific application or team.\\n\\n7. **Alerting and Notifications**: It can set up alerts for specific conditions or errors, notifying developers when something goes wrong or when performance metrics fall outside of acceptable ranges.\\n\\nBy providing comprehensive observability for NLP and LLM applications, Langfuse helps developers to build more reliable, accurate, and user-friendly models and services.\", refusal=None, role='assistant', audio=None, function_call=None, tool_calls=None))], created=1739554005, model='us.amazon.nova-micro-v1:0', object='chat.completion', service_tier=None, system_fingerprint=None, usage=CompletionUsage(completion_tokens=380, prompt_tokens=5, total_tokens=385, completion_tokens_details=None, prompt_tokens_details=None))"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import openai\n",
"client = openai.OpenAI(\n",
" api_key=LITELLM_VIRTUAL_KEY,\n",
" base_url=LITELLM_PROXY_BASE_URL\n",
")\n",
"\n",
"response = client.chat.completions.create(\n",
" model=\"us.amazon.nova-micro-v1:0\",\n",
" messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"what is Langfuse?\"\n",
" }\n",
" ],\n",
")\n",
"\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 3. Advanced - Set Langfuse Trace ID, Tags, Metadata \n",
"\n",
"Here is an example of how you can set Langfuse specific params on your client side request. See full list of supported langfuse params [here](https://docs.litellm.ai/docs/observability/langfuse_integration)\n",
"\n",
"You can view the logged trace of this request [here](https://us.cloud.langfuse.com/project/clvlhdfat0007vwb74m9lvfvi/traces/567890?timestamp=2025-02-14T17%3A30%3A26.709Z)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"ChatCompletion(id='chatcmpl-789babd5-c064-4939-9093-46e4cd2e208a', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content=\"Langfuse is an observability platform designed specifically for monitoring and improving the performance of natural language processing (NLP) models and applications. It provides developers with tools to track, analyze, and optimize how their language models interact with users and handle natural language inputs.\\n\\nHere are some key features and benefits of Langfuse:\\n\\n1. **Real-Time Monitoring**: Langfuse allows developers to monitor their NLP applications in real time. This includes tracking user interactions, model responses, and overall performance metrics.\\n\\n2. **Error Tracking**: It helps in identifying and tracking errors in the model's responses. This can include incorrect, irrelevant, or unsafe outputs.\\n\\n3. **User Feedback Integration**: Langfuse enables the collection of user feedback directly within the platform. This feedback can be used to identify areas for improvement in the model's performance.\\n\\n4. **Performance Metrics**: The platform provides detailed metrics and analytics on model performance, including latency, throughput, and accuracy.\\n\\n5. **Alerts and Notifications**: Developers can set up alerts to notify them of any significant issues or anomalies in model performance.\\n\\n6. **Debugging Tools**: Langfuse offers tools to help developers debug and refine their models by providing insights into how the model processes different types of inputs.\\n\\n7. **Integration with Development Workflows**: It integrates seamlessly with various development environments and CI/CD pipelines, making it easier to incorporate observability into the development process.\\n\\n8. **Customizable Dashboards**: Users can create custom dashboards to visualize the data in a way that best suits their needs.\\n\\nLangfuse aims to help developers build more reliable, accurate, and user-friendly NLP applications by providing them with the tools to observe and improve how their models perform in real-world scenarios.\", refusal=None, role='assistant', audio=None, function_call=None, tool_calls=None))], created=1739554281, model='us.amazon.nova-micro-v1:0', object='chat.completion', service_tier=None, system_fingerprint=None, usage=CompletionUsage(completion_tokens=346, prompt_tokens=5, total_tokens=351, completion_tokens_details=None, prompt_tokens_details=None))"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import openai\n",
"client = openai.OpenAI(\n",
" api_key=LITELLM_VIRTUAL_KEY,\n",
" base_url=LITELLM_PROXY_BASE_URL\n",
")\n",
"\n",
"response = client.chat.completions.create(\n",
" model=\"us.amazon.nova-micro-v1:0\",\n",
" messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"what is Langfuse?\"\n",
" }\n",
" ],\n",
" extra_body={\n",
" \"metadata\": {\n",
" \"generation_id\": \"1234567890\",\n",
" \"trace_id\": \"567890\",\n",
" \"trace_user_id\": \"user_1234567890\",\n",
" \"tags\": [\"tag1\", \"tag2\"]\n",
" }\n",
" }\n",
")\n",
"\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## "
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

BIN
cookbook/logging_observability/litellm_proxy_langfuse.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 308 KiB

517
cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md Normal file
View file

@ -0,0 +1,517 @@
# LiteLLM Release Notes Generation Instructions
This document provides comprehensive instructions for AI agents to generate release notes for LiteLLM following the established format and style.
## Required Inputs
1. **Release Version** (e.g., `v1.77.3-stable`)
2. **PR Diff/Changelog** - List of PRs with titles and contributors
3. **Previous Version Commit Hash** - To compare model pricing changes
4. **Reference Release Notes** - Use recent stable releases (v1.76.3-stable, v1.77.2-stable) as templates for consistent formatting
## Step-by-Step Process
### 1. Initial Setup and Analysis
```bash
# Check git diff for model pricing changes
git diff <previous_commit_hash> HEAD -- model_prices_and_context_window.json
```
**Key Analysis Points:**
- New models added (look for new entries)
- Deprecated models removed (look for deleted entries)
- Pricing updates (look for cost changes)
- Feature support changes (tool calling, reasoning, etc.)
### 2. Release Notes Structure
Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable, v1.77.5-stable):
```markdown
---
title: "v1.77.X-stable - [Key Theme]"
slug: "v1-77-X"
date: YYYY-MM-DDTHH:mm:ss
authors: [standard author block]
hide_table_of_contents: false
---
## Deploy this version
[Docker and pip installation tabs]
## Key Highlights
[3-5 bullet points of major features - prioritize MCP OAuth 2.0, scheduled key rotations, and major model updates]
## New Providers and Endpoints
### New Providers
[Table with Provider, Supported Endpoints, Description columns]
### New LLM API Endpoints
[Optional table for new endpoint additions with Endpoint, Method, Description, Documentation columns]
## New Models / Updated Models
#### New Model Support
[Model pricing table]
#### Features
[Provider-specific features organized by provider]
### Bug Fixes
[Provider-specific bug fixes organized by provider]
## LLM API Endpoints
#### Features
[API-specific features organized by API type]
#### Bugs
[General bug fixes]
## Management Endpoints / UI
#### Features
[UI and management features - group by functionality like Proxy CLI Auth, Virtual Keys, Models + Endpoints]
#### Bugs
[Management-related bug fixes]
## AI Integrations
### Logging
[Logging integrations organized by provider with proper doc links, includes General subsection]
### Guardrails
[Guardrail-specific features and fixes]
### Prompt Management
[Prompt management integrations like BitBucket]
### Secret Managers
[Secret manager integrations - AWS, HashiCorp Vault, CyberArk, etc.]
## Spend Tracking, Budgets and Rate Limiting
[Cost tracking, service tier pricing, rate limiting improvements]
## MCP Gateway
[MCP-specific features, OAuth 2.0, configuration improvements]
## Performance / Loadbalancing / Reliability improvements
[Infrastructure improvements, memory fixes, performance optimizations]
## Documentation Updates
[Documentation improvements, guides, corrections - separate section for visibility]
## New Contributors
[List of first-time contributors]
## Full Changelog
[Link to GitHub comparison]
```
### 3. Categorization Rules
**Performance Improvements:**
- RPS improvements
- Memory optimizations
- CPU usage optimizations
- Timeout controls
- Worker configuration
- Memory leak fixes
- Cache performance improvements
- Database connection management
- Dependency management (fastuuid, etc.)
- Configuration management
**New Models/Updated Models:**
- Extract from model_prices_and_context_window.json diff
- Create tables with: Provider, Model, Context Window, Input Cost, Output Cost, Features
- **Structure:**
- `#### New Model Support` - pricing table
- `#### Features` - organized by provider with documentation links
- `### Bug Fixes` - provider-specific bug fixes
- `#### New Provider Support` - major new provider integrations
- Group by provider with proper doc links: `**[Provider Name](../../docs/providers/[provider])**`
- Use bullet points under each provider for multiple features
- Separate features from bug fixes clearly
**LLM API Endpoints:**
- **Structure:**
- `#### Features` - organized by API type (Responses API, Batch API, etc.)
- `#### Bugs` - general bug fixes under **General** category
- **API Categories:**
- Responses API
- Batch API
- CountTokens API
- Images API
- Video Generation (if applicable)
- General (miscellaneous improvements)
- Use proper documentation links for each API type
**UI/Management:**
- Authentication changes
- Dashboard improvements
- Team management
- Key management
- Proxy CLI authentication and improvements
- Virtual key management and scheduled rotations
- SSO configuration fixes
- Admin settings updates
- Management routes and endpoints
**AI Integrations:**
- **Structure:**
- `### Logging` - organized by integration provider with proper doc links, includes **General** subsection
- `### Guardrails` - guardrail-specific features and fixes
- `### Prompt Management` - prompt management integrations
- `### Secret Managers` - secret manager integrations
- **Logging Categories:**
- **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes
- **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features
- **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements
- **[PostHog](../../docs/observability/posthog)** - observability integration
- **[SQS](../../docs/proxy/logging#sqs)** - SQS logging features
- **[Opik](../../docs/proxy/logging#opik)** - Opik integration improvements
- **[Arize Phoenix](../../docs/observability/arize_phoenix)** - Arize Phoenix integration
- **General** - miscellaneous logging features like callback controls, sensitive data masking
- Other logging providers with proper doc links
- **Guardrail Categories:**
- LakeraAI, Presidio, Noma, Grayswan, IBM Guardrails, and other guardrail providers
- **Prompt Management:**
- BitBucket, GitHub, and other prompt management integrations
- Prompt versioning, testing, and UI features
- **Secret Managers:**
- **[AWS Secrets Manager](../../docs/secret_managers)** - AWS secret manager features
- **[HashiCorp Vault](../../docs/secret_managers)** - Vault integrations
- **[CyberArk](../../docs/secret_managers)** - CyberArk integrations
- **General** - cross-secret-manager features
- Use bullet points under each provider for multiple features
- Separate logging, guardrails, prompt management, and secret managers clearly
### 4. Documentation Linking Strategy
**Link to docs when:**
- New provider support added
- Significant feature additions
- API endpoint changes
- Integration additions
**Link format:** `../../docs/[category]/[specific_doc]`
**Common doc paths:**
- `../../docs/providers/[provider]` - Provider-specific docs
- `../../docs/image_generation` - Image generation
- `../../docs/video_generation` - Video generation (if exists)
- `../../docs/response_api` - Responses API
- `../../docs/proxy/logging` - Logging integrations
- `../../docs/proxy/guardrails` - Guardrails
- `../../docs/pass_through/[provider]` - Passthrough endpoints
### 5. Model Table Generation
From git diff analysis, create tables like:
```markdown
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| OpenRouter | `openrouter/openai/gpt-4.1` | 1M | $2.00 | $8.00 | Chat completions with vision |
```
**Extract from JSON:**
- `max_input_tokens` → Context Window
- `input_cost_per_token` × 1,000,000 → Input cost
- `output_cost_per_token` × 1,000,000 → Output cost
- `supports_*` fields → Features
- Special pricing fields (per image, per second) for generation models
### 6. PR Categorization Logic
**By Keywords in PR Title:**
- `[Perf]`, `Performance`, `RPS` → Performance Improvements
- `[Bug]`, `[Bug Fix]`, `Fix` → Bug Fixes section
- `[Feat]`, `[Feature]`, `Add support` → Features section
- `[Docs]` → Documentation Updates section
- Provider names (Gemini, OpenAI, etc.) → Group under provider
- `MCP`, `oauth`, `Model Context Protocol` → MCP Gateway
- `service_tier`, `priority`, `cost tracking` → Spend Tracking, Budgets and Rate Limiting
**By PR Content Analysis:**
- New model additions → New Models section
- UI changes → Management Endpoints/UI
- Logging/observability → Logging/Guardrail/Prompt Management Integrations
- Rate limiting/budgets → Spend Tracking, Budgets and Rate Limiting
- Authentication → Management Endpoints/UI
- MCP-related changes → MCP Gateway
- Documentation updates → Documentation Updates
- Performance/memory fixes → Performance/Loadbalancing/Reliability improvements
**Special Categorization Rules:**
- **Service tier pricing** (OpenAI priority/flex) → Spend Tracking section (NOT provider features)
- **Cost breakdown in logging** → Spend Tracking section
- **MCP configuration/OAuth** → MCP Gateway (NOT General Proxy Improvements)
- **All documentation PRs** → Documentation Updates section for visibility
- **Callback controls/logging features** → AI Integrations > Logging > General
- **Secret manager features** → AI Integrations > Secret Managers
- **Video generation tag-based routing** → LLM API Endpoints > Video Generation API
### 7. Writing Style Guidelines
**Tone:**
- Professional but accessible
- Focus on user impact
- Highlight breaking changes clearly
- Use active voice
**Formatting:**
- Use consistent markdown formatting
- Include PR links: `[PR #XXXXX](https://github.com/BerriAI/litellm/pull/XXXXX)`
- Use code blocks for configuration examples
- Bold important terms and section headers
**Warnings/Notes:**
- Add warning boxes for breaking changes
- Include migration instructions when needed
- Provide override options for default changes
### 8. Quality Checks
**Before finalizing:**
- Verify all PR links work
- Check documentation links are valid
- Ensure model pricing is accurate
- Confirm provider names are consistent
- Review for typos and formatting issues
- **Count PRs by section** - Provide final count like:
```
## MM/DD/YYYY
* New Models / Updated Models: XX
* LLM API Endpoints: XX
* Management Endpoints / UI: XX
* Logging / Guardrail / Prompt Management Integrations: XX
* Spend Tracking, Budgets and Rate Limiting: XX
* MCP Gateway: XX
* Performance / Loadbalancing / Reliability improvements: XX
* Documentation Updates: XX
```
### 9. Common Patterns to Follow
**Performance Changes:**
```markdown
- **+400 RPS Performance Boost** - Description - [PR #XXXXX](link)
```
**New Models:**
Always include pricing table and feature highlights
**Breaking Changes:**
```markdown
:::warning
This release has a known issue...
:::
```
**Provider Features (New Models / Updated Models section):**
```markdown
#### Features
- **[Provider Name](../../docs/providers/provider)**
- Feature description - [PR #XXXXX](link)
- Another feature description - [PR #YYYYY](link)
```
**API Features (LLM API Endpoints section):**
```markdown
#### Features
- **[API Name](../../docs/api_path)**
- Feature description - [PR #XXXXX](link)
- Another feature - [PR #YYYYY](link)
- **General**
- Miscellaneous improvements - [PR #ZZZZZ](link)
```
**Integration Features (Logging / Guardrail Integrations section):**
```markdown
#### Features
- **[Integration Name](../../docs/proxy/logging#integration)**
- Feature description - [PR #XXXXX](link)
- Bug fix description - [PR #YYYYY](link)
```
**Bug Fixes Pattern:**
```markdown
### Bug Fixes
- **[Provider/Component Name](../../docs/providers/provider)**
- Bug fix description - [PR #XXXXX](link)
```
### 10. Missing Documentation Check
**Review for missing docs:**
- New providers without documentation
- New API endpoints without examples
- Complex features without guides
- Integration setup instructions
**Flag for documentation needs:**
- New provider integrations
- Significant API changes
- Complex configuration options
- Migration requirements
### 11. New Sections and Categories (Added in v1.77.5)
**MCP Gateway Section:**
- All MCP-related changes go here (not in General Proxy Improvements)
- OAuth 2.0 flow improvements
- MCP configuration and tools
- Server management features
**Spend Tracking, Budgets and Rate Limiting Section:**
- Service tier pricing (OpenAI priority/flex pricing)
- Cost tracking and breakdown features
- Rate limiting improvements (Parallel Request Limiter v3)
- Priority reservation fixes
- Metadata handling for rate limiting
**Documentation Updates Section:**
- Create separate section for all documentation improvements
- Include provider documentation fixes
- Model reference updates
- New guides and tutorials
- Documentation corrections and clarifications
- This gives documentation changes proper visibility
**Management Endpoints / UI Grouping:**
- Group related features under sub-categories:
- **Proxy CLI Auth** - CLI authentication improvements
- **Virtual Keys** - Key rotation and management
- **Models + Endpoints** - Provider and endpoint management
**AI Integrations Section Expansion:**
- Renamed from "Logging / Guardrail / Prompt Management Integrations" to "AI Integrations"
- Structure with four main subsections:
- **Logging** - with **General** subsection for miscellaneous logging features
- **Guardrails** - separate from logging features
- **Prompt Management** - BitBucket, GitHub integrations, versioning features
- **Secret Managers** - AWS, HashiCorp Vault, CyberArk, etc.
**New Providers and Endpoints Section:**
- Add section after Key Highlights and before New Models / Updated Models
- Include tables for:
- **New Providers** - Provider name, supported endpoints, description
- **New LLM API Endpoints** (optional) - Endpoint, method, description, documentation link
- Only include major new provider integrations, not minor provider updates
- **IMPORTANT**: When adding new providers, also update `provider_endpoints_support.json` in the repository root (see Section 13)
### 12. Section Header Counts
**Always include counts in section headers for:**
- **New Providers** - Add count in parentheses: `### New Providers (X new providers)`
- **New LLM API Endpoints** - Add count in parentheses: `### New LLM API Endpoints (X new endpoints)`
- **New Model Support** - Add count in parentheses: `#### New Model Support (X new models)`
**Format:**
```markdown
### New Providers (4 new providers)
| Provider | Supported LiteLLM Endpoints | Description |
| -------- | --------------------------- | ----------- |
...
### New LLM API Endpoints (2 new endpoints)
| Endpoint | Method | Description | Documentation |
| -------- | ------ | ----------- | ------------- |
...
#### New Model Support (32 new models)
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
...
```
**Counting Rules:**
- Count each row in the table (excluding the header row)
- For models, count each model entry in the pricing table
- For providers, count each new provider added
- For endpoints, count each new API endpoint added
### 13. Update provider_endpoints_support.json
**When adding new providers or endpoints, you MUST also update `provider_endpoints_support.json` in the repository root.**
This file tracks which endpoints are supported by each LiteLLM provider and is used to generate documentation.
**Required Steps:**
1. For each new provider added to the release notes, add a corresponding entry to `provider_endpoints_support.json`
2. For each new endpoint type added, update the schema comment and add the endpoint to relevant providers
**Provider Entry Format:**
```json
"provider_slug": {
"display_name": "Provider Name (`provider_slug`)",
"url": "https://docs.litellm.ai/docs/providers/provider_slug",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": true
}
}
```
**Available Endpoint Types:**
- `chat_completions` - `/chat/completions` endpoint
- `messages` - `/messages` endpoint (Anthropic format)
- `responses` - `/responses` endpoint (OpenAI/Anthropic unified)
- `embeddings` - `/embeddings` endpoint
- `image_generations` - `/image/generations` endpoint
- `audio_transcriptions` - `/audio/transcriptions` endpoint
- `audio_speech` - `/audio/speech` endpoint
- `moderations` - `/moderations` endpoint
- `batches` - `/batches` endpoint
- `rerank` - `/rerank` endpoint
- `ocr` - `/ocr` endpoint
- `search` - `/search` endpoint
- `vector_stores` - `/vector_stores` endpoint
- `a2a` - `/a2a/{agent}/message/send` endpoint (A2A Protocol)
**Checklist:**
- [ ] All new providers from release notes are added to `provider_endpoints_support.json`
- [ ] Endpoint support flags accurately reflect provider capabilities
- [ ] Documentation URL points to correct provider docs page
## Example Command Workflow
```bash
# 1. Get model changes
git diff <commit> HEAD -- model_prices_and_context_window.json
# 2. Analyze PR list for categorization
# 3. Create release notes following template
# 4. Link to appropriate documentation
# 5. Review for missing documentation needs
```
## Output Requirements
- Follow exact markdown structure from reference
- Include all PR links and contributors
- Provide accurate model pricing tables
- Link to relevant documentation
- Highlight breaking changes with warnings
- Include deployment instructions
- End with full changelog link
This process ensures consistent, comprehensive release notes that help users understand changes and upgrade smoothly.

11
cookbook/misc/dev_release.txt
View file

@ -1,2 +1,11 @@
python3 -m build
twine upload --verbose dist/litellm-1.18.13.dev4.tar.gz -u __token__ -
twine upload --verbose dist/litellm-1.18.13.dev4.tar.gz -u __token__ -
Note: You might need to make a MANIFEST.ini file on root for build process incase it fails
Place this in MANIFEST.ini
recursive-exclude venv *
recursive-exclude myenv *
recursive-exclude py313_env *
recursive-exclude **/.venv *

53
cookbook/misc/test_responses_api.py Normal file
View file

@ -0,0 +1,53 @@
import base64
from openai import OpenAI
import time
client = OpenAI(
base_url="http://0.0.0.0:4001",
api_key="sk-1234"
)
# Function to encode the image
def encode_image(image_path):
with open(image_path, "rb") as image_file:
return base64.b64encode(image_file.read()).decode("utf-8")
# Path to your image
image_path = "litellm/proxy/logo.jpg"
# Getting the Base64 string
base64_image = encode_image(image_path)
response = client.responses.create(
model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
input=[
{
"role": "user",
"content": [
{ "type": "input_text", "text": "what color is the image"},
{
"type": "input_image",
"image_url": f"data:image/jpeg;base64,{base64_image}",
},
],
}
],
)
print(response.output_text)
print("response1 id===", response.id)
print("sleeping for 20 seconds...")
time.sleep(20)
print("making follow up request for existing id")
response2 = client.responses.create(
model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
previous_response_id=response.id,
input="ok, and what objects are in the image?"
)
print(response2.output_text)

540
cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py Normal file
View file

@ -0,0 +1,540 @@
#!/usr/bin/env python3
"""
Mock Bedrock Guardrail API Server
This is a FastAPI server that mimics the AWS Bedrock Guardrail API for testing purposes.
It follows the same API spec as the real Bedrock guardrail endpoint.
Usage:
python mock_bedrock_guardrail_server.py
The server will start on http://localhost:8080
"""
import os
import re
from typing import Any, Dict, List, Literal, Optional
from fastapi import Depends, FastAPI, Header, HTTPException, status
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field
# ============================================================================
# Request/Response Models (matching Bedrock API spec)
# ============================================================================
class BedrockTextContent(BaseModel):
text: str
class BedrockContentItem(BaseModel):
text: BedrockTextContent
class BedrockRequest(BaseModel):
source: Literal["INPUT", "OUTPUT"]
content: List[BedrockContentItem] = Field(default_factory=list)
class BedrockGuardrailOutput(BaseModel):
text: Optional[str] = None
class TopicPolicyItem(BaseModel):
name: str
type: str
action: Literal["BLOCKED", "NONE"]
class TopicPolicy(BaseModel):
topics: List[TopicPolicyItem] = Field(default_factory=list)
class ContentFilterItem(BaseModel):
type: str
confidence: str
action: Literal["BLOCKED", "NONE"]
class ContentPolicy(BaseModel):
filters: List[ContentFilterItem] = Field(default_factory=list)
class CustomWord(BaseModel):
match: str
action: Literal["BLOCKED", "NONE"]
class WordPolicy(BaseModel):
customWords: List[CustomWord] = Field(default_factory=list)
managedWordLists: List[Dict[str, Any]] = Field(default_factory=list)
class PiiEntity(BaseModel):
type: str
match: str
action: Literal["BLOCKED", "ANONYMIZED", "NONE"]
class RegexMatch(BaseModel):
name: str
match: str
regex: str
action: Literal["BLOCKED", "ANONYMIZED", "NONE"]
class SensitiveInformationPolicy(BaseModel):
piiEntities: List[PiiEntity] = Field(default_factory=list)
regexes: List[RegexMatch] = Field(default_factory=list)
class ContextualGroundingFilter(BaseModel):
type: str
threshold: float
score: float
action: Literal["BLOCKED", "NONE"]
class ContextualGroundingPolicy(BaseModel):
filters: List[ContextualGroundingFilter] = Field(default_factory=list)
class Assessment(BaseModel):
topicPolicy: Optional[TopicPolicy] = None
contentPolicy: Optional[ContentPolicy] = None
wordPolicy: Optional[WordPolicy] = None
sensitiveInformationPolicy: Optional[SensitiveInformationPolicy] = None
contextualGroundingPolicy: Optional[ContextualGroundingPolicy] = None
class BedrockGuardrailResponse(BaseModel):
usage: Dict[str, int] = Field(
default_factory=lambda: {"topicPolicyUnits": 1, "contentPolicyUnits": 1}
)
action: Literal["NONE", "GUARDRAIL_INTERVENED"] = "NONE"
outputs: List[BedrockGuardrailOutput] = Field(default_factory=list)
assessments: List[Assessment] = Field(default_factory=list)
# ============================================================================
# Mock Guardrail Configuration
# ============================================================================
class GuardrailConfig(BaseModel):
"""Configuration for mock guardrail behavior"""
blocked_words: List[str] = Field(
default_factory=lambda: ["offensive", "inappropriate", "badword"]
)
blocked_topics: List[str] = Field(default_factory=lambda: ["violence", "illegal"])
pii_patterns: Dict[str, str] = Field(
default_factory=lambda: {
"EMAIL": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",
"PHONE": r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b",
"SSN": r"\b\d{3}-\d{2}-\d{4}\b",
"CREDIT_CARD": r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b",
}
)
anonymize_pii: bool = True # If True, ANONYMIZE PII; if False, BLOCK it
bearer_token: str = "mock-bedrock-token-12345"
# Global config
GUARDRAIL_CONFIG = GuardrailConfig()
# ============================================================================
# FastAPI App Setup
# ============================================================================
app = FastAPI(
title="Mock Bedrock Guardrail API",
description="Mock server mimicking AWS Bedrock Guardrail API",
version="1.0.0",
)
# ============================================================================
# Authentication
# ============================================================================
async def verify_bearer_token(authorization: Optional[str] = Header(None)) -> str:
"""
Verify the Bearer token from the Authorization header.
Args:
authorization: The Authorization header value
Returns:
The token if valid
Raises:
HTTPException: If token is missing or invalid
"""
if authorization is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Missing Authorization header",
headers={"WWW-Authenticate": "Bearer"},
)
# Check if it's a Bearer token
parts = authorization.split()
print(f"parts: {parts}")
if len(parts) != 2 or parts[0].lower() != "bearer":
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid Authorization header format. Expected: Bearer <token>",
headers={"WWW-Authenticate": "Bearer"},
)
token = parts[1]
# Verify token
if token != GUARDRAIL_CONFIG.bearer_token:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Invalid bearer token",
)
return token
# ============================================================================
# Guardrail Logic
# ============================================================================
def check_blocked_words(text: str) -> Optional[WordPolicy]:
"""Check if text contains blocked words"""
found_words = []
text_lower = text.lower()
for word in GUARDRAIL_CONFIG.blocked_words:
if word.lower() in text_lower:
found_words.append(CustomWord(match=word, action="BLOCKED"))
if found_words:
return WordPolicy(customWords=found_words)
return None
def check_blocked_topics(text: str) -> Optional[TopicPolicy]:
"""Check if text contains blocked topics"""
found_topics = []
text_lower = text.lower()
for topic in GUARDRAIL_CONFIG.blocked_topics:
if topic.lower() in text_lower:
found_topics.append(
TopicPolicyItem(name=topic, type=topic.upper(), action="BLOCKED")
)
if found_topics:
return TopicPolicy(topics=found_topics)
return None
def check_pii(text: str) -> tuple[Optional[SensitiveInformationPolicy], str]:
"""
Check for PII in text and return policy + anonymized text
Returns:
Tuple of (SensitiveInformationPolicy or None, anonymized_text)
"""
pii_entities = []
anonymized_text = text
action = "ANONYMIZED" if GUARDRAIL_CONFIG.anonymize_pii else "BLOCKED"
for pii_type, pattern in GUARDRAIL_CONFIG.pii_patterns.items():
try:
# Compile the regex pattern with a timeout to prevent ReDoS attacks
compiled_pattern = re.compile(pattern)
matches = compiled_pattern.finditer(text)
for match in matches:
matched_text = match.group()
pii_entities.append(
PiiEntity(type=pii_type, match=matched_text, action=action)
)
# Anonymize the text if configured
if GUARDRAIL_CONFIG.anonymize_pii:
anonymized_text = anonymized_text.replace(
matched_text, f"[{pii_type}_REDACTED]"
)
except re.error:
# Invalid regex pattern - skip it and log a warning
print(f"Warning: Invalid regex pattern for PII type {pii_type}: {pattern}")
continue
if pii_entities:
return SensitiveInformationPolicy(piiEntities=pii_entities), anonymized_text
return None, text
def process_guardrail_request(
request: BedrockRequest,
) -> tuple[BedrockGuardrailResponse, List[str]]:
"""
Process a guardrail request and return the response.
Returns:
Tuple of (response, list of output texts)
"""
all_text_content = []
output_texts = []
# Extract all text from content items
for content_item in request.content:
if content_item.text and content_item.text.text:
all_text_content.append(content_item.text.text)
# Combine all text for analysis
combined_text = " ".join(all_text_content)
# Initialize response
response = BedrockGuardrailResponse()
assessment = Assessment()
has_intervention = False
# Check for blocked words
word_policy = check_blocked_words(combined_text)
if word_policy:
assessment.wordPolicy = word_policy
has_intervention = True
# Check for blocked topics
topic_policy = check_blocked_topics(combined_text)
if topic_policy:
assessment.topicPolicy = topic_policy
has_intervention = True
# Check for PII
for text in all_text_content:
pii_policy, anonymized_text = check_pii(text)
if pii_policy:
assessment.sensitiveInformationPolicy = pii_policy
if GUARDRAIL_CONFIG.anonymize_pii:
# If anonymizing, we don't block, we modify the text
output_texts.append(anonymized_text)
has_intervention = True
else:
# If not anonymizing PII, we block it
output_texts.append(text)
has_intervention = True
else:
output_texts.append(text)
# Build response
if has_intervention:
response.action = "GUARDRAIL_INTERVENED"
# Only add assessment if there were interventions
response.assessments = [assessment]
# Add outputs (modified or original text)
response.outputs = [BedrockGuardrailOutput(text=txt) for txt in output_texts]
return response, output_texts
# ============================================================================
# API Endpoints
# ============================================================================
@app.get("/")
async def root():
"""Health check endpoint"""
return {
"service": "Mock Bedrock Guardrail API",
"status": "running",
"endpoint_format": "/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply",
}
@app.get("/health")
async def health():
"""Health check endpoint"""
return {"status": "healthy"}
"""
LiteLLM exposes a basic guardrail API with the text extracted from the request and sent to the guardrail API, as well as the received request body for any further processing.
This works across all LiteLLM endpoints (completion, anthropic /v1/messages, responses api, image generation, embedding, etc.)
This makes it easy to support your own guardrail API without having to make a PR to LiteLLM.
LiteLLM supports passing any provider specific params from LiteLLM config.yaml to the guardrail API.
Example:
```yaml
guardrails:
- guardrail_name: "bedrock-content-guard"
litellm_params:
guardrail: generic_guardrail_api
mode: "pre_call"
api_key: os.environ/GUARDRAIL_API_KEY
api_base: os.environ/GUARDRAIL_API_BASE
additional_provider_specific_params:
api_version: os.environ/GUARDRAIL_API_VERSION # additional provider specific params
```
This is a beta API. Please help us improve it.
"""
class LitellmBasicGuardrailRequest(BaseModel):
texts: List[str]
images: Optional[List[str]] = None
tools: Optional[List[dict]] = None
tool_calls: Optional[List[dict]] = None
request_data: Dict[str, Any] = Field(default_factory=dict)
additional_provider_specific_params: Dict[str, Any] = Field(default_factory=dict)
input_type: Literal["request", "response"]
litellm_call_id: Optional[str] = None
litellm_trace_id: Optional[str] = None
structured_messages: Optional[List[Dict[str, Any]]] = None
class LitellmBasicGuardrailResponse(BaseModel):
action: Literal[
"BLOCKED", "NONE", "GUARDRAIL_INTERVENED"
] # BLOCKED = litellm will raise an error, NONE = litellm will continue, GUARDRAIL_INTERVENED = litellm will continue, but the text was modified by the guardrail
blocked_reason: Optional[str] = None # only if action is BLOCKED, otherwise None
texts: Optional[List[str]] = None
images: Optional[List[str]] = None
@app.post(
"/beta/litellm_basic_guardrail_api",
response_model=LitellmBasicGuardrailResponse,
)
async def beta_litellm_basic_guardrail_api(
request: LitellmBasicGuardrailRequest,
) -> LitellmBasicGuardrailResponse:
"""
Apply guardrail to input or output content.
This endpoint mimics the AWS Bedrock ApplyGuardrail API.
Args:
request: The guardrail request containing content to analyze
token: Bearer token (verified by dependency)
Returns:
LitellmBasicGuardrailResponse with analysis results
"""
print(f"request: {request}")
if any("ishaan" in text.lower() for text in request.texts):
return LitellmBasicGuardrailResponse(
action="BLOCKED", blocked_reason="Ishaan is not allowed"
)
elif any("pii_value" in text for text in request.texts):
return LitellmBasicGuardrailResponse(
action="GUARDRAIL_INTERVENED",
texts=[
text.replace("pii_value", "pii_value_redacted")
for text in request.texts
],
)
return LitellmBasicGuardrailResponse(action="NONE")
@app.post("/config/update")
async def update_config(
config: GuardrailConfig, token: str = Depends(verify_bearer_token)
):
"""
Update the guardrail configuration.
This is a testing endpoint to modify the mock guardrail behavior.
Args:
config: New guardrail configuration
token: Bearer token (verified by dependency)
Returns:
Updated configuration
"""
global GUARDRAIL_CONFIG
GUARDRAIL_CONFIG = config
return {"status": "updated", "config": GUARDRAIL_CONFIG}
@app.get("/config")
async def get_config(token: str = Depends(verify_bearer_token)):
"""
Get the current guardrail configuration.
Args:
token: Bearer token (verified by dependency)
Returns:
Current configuration
"""
return GUARDRAIL_CONFIG
# ============================================================================
# Error Handlers
# ============================================================================
@app.exception_handler(HTTPException)
async def http_exception_handler(request, exc: HTTPException):
"""Custom error handler for HTTP exceptions"""
return JSONResponse(
status_code=exc.status_code,
content={"error": exc.detail},
headers=exc.headers,
)
# ============================================================================
# Main
# ============================================================================
if __name__ == "__main__":
import uvicorn
# Get configuration from environment
host = os.getenv("MOCK_BEDROCK_HOST", "0.0.0.0")
port = int(os.getenv("MOCK_BEDROCK_PORT", "8080"))
bearer_token = os.getenv("MOCK_BEDROCK_TOKEN", "mock-bedrock-token-12345")
# Update config with environment token
GUARDRAIL_CONFIG.bearer_token = bearer_token
print("=" * 80)
print("Mock Bedrock Guardrail API Server")
print("=" * 80)
print(f"Server starting on: http://{host}:{port}")
print(f"Bearer Token: {bearer_token}")
print(f"Endpoint: POST /guardrail/{{id}}/version/{{version}}/apply")
print("=" * 80)
print("\nExample curl command:")
print(
f"""
curl -X POST "http://{host}:{port}/guardrail/test-guardrail/version/1/apply" \\
-H "Authorization: Bearer {bearer_token}" \\
-H "Content-Type: application/json" \\
-d '{{
"source": "INPUT",
"content": [
{{
"text": {{
"text": "Hello, my email is test@example.com"
}}
}}
]
}}'
"""
)
print("=" * 80)
uvicorn.run(app, host=host, port=port)

284
cookbook/nova_sonic_realtime.py Normal file
View file

@ -0,0 +1,284 @@
"""
Client script to test Nova Sonic realtime API through LiteLLM proxy.
This script connects to LiteLLM proxy's realtime endpoint and enables
speech-to-speech conversation with Bedrock Nova Sonic.
Prerequisites:
- LiteLLM proxy running with Bedrock configured
- pyaudio installed: pip install pyaudio
- websockets installed: pip install websockets
Usage:
python nova_sonic_realtime.py
"""
import asyncio
import base64
import json
import pyaudio
import websockets
from typing import Optional
# Audio configuration (matching Nova Sonic requirements)
INPUT_SAMPLE_RATE = 16000 # Nova Sonic expects 16kHz input
OUTPUT_SAMPLE_RATE = 24000 # Nova Sonic outputs 24kHz
CHANNELS = 1
FORMAT = pyaudio.paInt16
CHUNK_SIZE = 1024
# LiteLLM proxy configuration
LITELLM_PROXY_URL = "ws://localhost:4000/v1/realtime?model=bedrock-sonic"
LITELLM_API_KEY = "sk-12345" # Your LiteLLM API key
class RealtimeClient:
"""Client for LiteLLM realtime API with audio support."""
def __init__(self, url: str, api_key: str):
self.url = url
self.api_key = api_key
self.ws: Optional[websockets.WebSocketClientProtocol] = None
self.is_active = False
self.audio_queue = asyncio.Queue()
self.pyaudio = pyaudio.PyAudio()
self.input_stream = None
self.output_stream = None
async def connect(self):
"""Connect to LiteLLM proxy realtime endpoint."""
print(f"Connecting to {self.url}...")
headers = {}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
self.ws = await websockets.connect(
self.url,
additional_headers=headers,
max_size=10 * 1024 * 1024, # 10MB max message size
)
self.is_active = True
print("✓ Connected to LiteLLM proxy")
async def send_session_update(self):
"""Send session configuration."""
session_update = {
"type": "session.update",
"session": {
"instructions": "You are a friendly assistant. Keep your responses short and conversational.",
"voice": "matthew",
"temperature": 0.8,
"max_response_output_tokens": 1024,
"modalities": ["text", "audio"],
"input_audio_format": "pcm16",
"output_audio_format": "pcm16",
"turn_detection": {
"type": "server_vad",
"threshold": 0.5,
"prefix_padding_ms": 300,
"silence_duration_ms": 500,
},
},
}
await self.ws.send(json.dumps(session_update))
print("✓ Session configuration sent")
async def receive_messages(self):
"""Receive and process messages from the server."""
try:
async for message in self.ws:
if not self.is_active:
break
try:
data = json.loads(message)
event_type = data.get("type")
if event_type == "session.created":
print(f"✓ Session created: {data.get('session', {}).get('id')}")
elif event_type == "response.created":
print("🤖 Assistant is responding...")
elif event_type == "response.text.delta":
# Print text transcription
delta = data.get("delta", "")
print(delta, end="", flush=True)
elif event_type == "response.audio.delta":
# Queue audio for playback
audio_b64 = data.get("delta", "")
if audio_b64:
audio_bytes = base64.b64decode(audio_b64)
await self.audio_queue.put(audio_bytes)
elif event_type == "response.text.done":
print() # New line after text
elif event_type == "response.done":
print("✓ Response complete")
elif event_type == "error":
print(f"❌ Error: {data.get('error', {})}")
else:
# Debug: print other event types
print(f"[{event_type}]", end=" ")
except json.JSONDecodeError:
print(f"Failed to parse message: {message[:100]}")
except websockets.exceptions.ConnectionClosed:
print("\n✗ Connection closed")
except Exception as e:
print(f"\n✗ Error receiving messages: {e}")
finally:
self.is_active = False
async def send_audio_chunk(self, audio_bytes: bytes):
"""Send audio chunk to server."""
if not self.is_active or not self.ws:
return
audio_b64 = base64.b64encode(audio_bytes).decode("utf-8")
message = {
"type": "input_audio_buffer.append",
"audio": audio_b64,
}
await self.ws.send(json.dumps(message))
async def commit_audio_buffer(self):
"""Commit the audio buffer to trigger processing."""
if not self.is_active or not self.ws:
return
message = {"type": "input_audio_buffer.commit"}
await self.ws.send(json.dumps(message))
async def capture_audio(self):
"""Capture audio from microphone and send to server."""
print("\n🎤 Starting audio capture...")
print("Speak into your microphone. Press Ctrl+C to stop.\n")
self.input_stream = self.pyaudio.open(
format=FORMAT,
channels=CHANNELS,
rate=INPUT_SAMPLE_RATE,
input=True,
frames_per_buffer=CHUNK_SIZE,
)
try:
while self.is_active:
audio_data = self.input_stream.read(CHUNK_SIZE, exception_on_overflow=False)
await self.send_audio_chunk(audio_data)
await asyncio.sleep(0.01) # Small delay to prevent overwhelming
except Exception as e:
print(f"Error capturing audio: {e}")
finally:
if self.input_stream:
self.input_stream.stop_stream()
self.input_stream.close()
async def play_audio(self):
"""Play audio responses from the server."""
print("🔊 Starting audio playback...")
self.output_stream = self.pyaudio.open(
format=FORMAT,
channels=CHANNELS,
rate=OUTPUT_SAMPLE_RATE,
output=True,
frames_per_buffer=CHUNK_SIZE,
)
try:
while self.is_active:
try:
audio_data = await asyncio.wait_for(
self.audio_queue.get(), timeout=0.1
)
if audio_data:
self.output_stream.write(audio_data)
except asyncio.TimeoutError:
continue
except Exception as e:
print(f"Error playing audio: {e}")
finally:
if self.output_stream:
self.output_stream.stop_stream()
self.output_stream.close()
async def close(self):
"""Close the connection and cleanup."""
self.is_active = False
if self.ws:
await self.ws.close()
if self.input_stream:
self.input_stream.stop_stream()
self.input_stream.close()
if self.output_stream:
self.output_stream.stop_stream()
self.output_stream.close()
self.pyaudio.terminate()
print("\n✓ Connection closed")
async def main():
"""Main function to run the realtime client."""
print("=" * 80)
print("Bedrock Nova Sonic Realtime Client")
print("=" * 80)
print()
client = RealtimeClient(LITELLM_PROXY_URL, LITELLM_API_KEY)
try:
# Connect to server
await client.connect()
# Send session configuration
await client.send_session_update()
# Wait a moment for session to be established
await asyncio.sleep(0.5)
# Start tasks
receive_task = asyncio.create_task(client.receive_messages())
capture_task = asyncio.create_task(client.capture_audio())
playback_task = asyncio.create_task(client.play_audio())
# Wait for user to interrupt
await asyncio.gather(
receive_task,
capture_task,
playback_task,
return_exceptions=True,
)
except KeyboardInterrupt:
print("\n\n⚠ Interrupted by user")
except Exception as e:
print(f"\n❌ Error: {e}")
import traceback
traceback.print_exc()
finally:
await client.close()
if __name__ == "__main__":
print("\nMake sure:")
print("1. LiteLLM proxy is running on port 4000")
print("2. Bedrock is configured in proxy_server_config.yaml")
print("3. AWS credentials are set")
print()
try:
asyncio.run(main())
except KeyboardInterrupt:
print("\n\nGoodbye!")

311
cookbook/veo_video_generation.py Normal file
View file

@ -0,0 +1,311 @@
#!/usr/bin/env python3
"""
Complete example for Veo video generation through LiteLLM proxy.
This script demonstrates how to:
1. Generate videos using Google's Veo model
2. Poll for completion status
3. Download the generated video file
Requirements:
- LiteLLM proxy running with Google AI Studio pass-through configured
- Google AI Studio API key with Veo access
"""
import json
import os
import time
import requests
from typing import Optional
class VeoVideoGenerator:
"""Complete Veo video generation client using LiteLLM proxy."""
def __init__(self, base_url: str = "http://localhost:4000/gemini/v1beta",
api_key: str = "sk-1234"):
"""
Initialize the Veo video generator.
Args:
base_url: Base URL for the LiteLLM proxy with Gemini pass-through
api_key: API key for LiteLLM proxy authentication
"""
self.base_url = base_url
self.api_key = api_key
self.headers = {
"x-goog-api-key": api_key,
"Content-Type": "application/json"
}
def generate_video(self, prompt: str) -> Optional[str]:
"""
Initiate video generation with Veo.
Args:
prompt: Text description of the video to generate
Returns:
Operation name if successful, None otherwise
"""
print(f"🎬 Generating video with prompt: '{prompt}'")
url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning"
payload = {
"instances": [{
"prompt": prompt
}]
}
try:
response = requests.post(url, headers=self.headers, json=payload)
response.raise_for_status()
data = response.json()
operation_name = data.get("name")
if operation_name:
print(f"✅ Video generation started: {operation_name}")
return operation_name
else:
print("❌ No operation name returned")
print(f"Response: {json.dumps(data, indent=2)}")
return None
except requests.RequestException as e:
print(f"❌ Failed to start video generation: {e}")
if hasattr(e, 'response') and e.response is not None:
try:
error_data = e.response.json()
print(f"Error details: {json.dumps(error_data, indent=2)}")
except:
print(f"Error response: {e.response.text}")
return None
def wait_for_completion(self, operation_name: str, max_wait_time: int = 600) -> Optional[str]:
"""
Poll operation status until video generation is complete.
Args:
operation_name: Name of the operation to monitor
max_wait_time: Maximum time to wait in seconds (default: 10 minutes)
Returns:
Video URI if successful, None otherwise
"""
print("⏳ Waiting for video generation to complete...")
operation_url = f"{self.base_url}/{operation_name}"
start_time = time.time()
poll_interval = 10 # Start with 10 seconds
while time.time() - start_time < max_wait_time:
try:
print(f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)")
response = requests.get(operation_url, headers=self.headers)
response.raise_for_status()
data = response.json()
# Check for errors
if "error" in data:
print("❌ Error in video generation:")
print(json.dumps(data["error"], indent=2))
return None
# Check if operation is complete
is_done = data.get("done", False)
if is_done:
print("🎉 Video generation complete!")
try:
# Extract video URI from nested response
video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
print(f"📹 Video URI: {video_uri}")
return video_uri
except KeyError as e:
print(f"❌ Could not extract video URI: {e}")
print("Full response:")
print(json.dumps(data, indent=2))
return None
# Wait before next poll, with exponential backoff
time.sleep(poll_interval)
poll_interval = min(poll_interval * 1.2, 30) # Cap at 30 seconds
except requests.RequestException as e:
print(f"❌ Error polling operation status: {e}")
time.sleep(poll_interval)
print(f"⏰ Timeout after {max_wait_time} seconds")
return None
def download_video(self, video_uri: str, output_filename: str = "generated_video.mp4") -> bool:
"""
Download the generated video file.
Args:
video_uri: URI of the video to download (from Google's response)
output_filename: Local filename to save the video
Returns:
True if download successful, False otherwise
"""
print(f"⬇️ Downloading video...")
print(f"Original URI: {video_uri}")
# Convert Google URI to LiteLLM proxy URI
# Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media
if video_uri.startswith("files/"):
download_path = f"{video_uri}:download?alt=media"
else:
download_path = video_uri
litellm_download_url = f"{self.base_url}/{download_path}"
print(f"Download URL: {litellm_download_url}")
try:
# Download with streaming and redirect handling
response = requests.get(
litellm_download_url,
headers=self.headers,
stream=True,
allow_redirects=True # Handle redirects automatically
)
response.raise_for_status()
# Save video file
with open(output_filename, 'wb') as f:
downloaded_size = 0
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
downloaded_size += len(chunk)
# Progress indicator for large files
if downloaded_size % (1024 * 1024) == 0: # Every MB
print(f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB...")
# Verify file was created and has content
if os.path.exists(output_filename):
file_size = os.path.getsize(output_filename)
if file_size > 0:
print(f"✅ Video downloaded successfully!")
print(f"📁 Saved as: {output_filename}")
print(f"📏 File size: {file_size / (1024*1024):.2f} MB")
return True
else:
print("❌ Downloaded file is empty")
os.remove(output_filename)
return False
else:
print("❌ File was not created")
return False
except requests.RequestException as e:
print(f"❌ Download failed: {e}")
if hasattr(e, 'response') and e.response is not None:
print(f"Status code: {e.response.status_code}")
print(f"Response headers: {dict(e.response.headers)}")
return False
def generate_and_download(self, prompt: str, output_filename: str = None) -> bool:
"""
Complete workflow: generate video and download it.
Args:
prompt: Text description for video generation
output_filename: Output filename (auto-generated if None)
Returns:
True if successful, False otherwise
"""
# Auto-generate filename if not provided
if output_filename is None:
timestamp = int(time.time())
safe_prompt = "".join(c for c in prompt[:30] if c.isalnum() or c in (' ', '-', '_')).rstrip()
output_filename = f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
print("=" * 60)
print("🎬 VEO VIDEO GENERATION WORKFLOW")
print("=" * 60)
# Step 1: Generate video
operation_name = self.generate_video(prompt)
if not operation_name:
return False
# Step 2: Wait for completion
video_uri = self.wait_for_completion(operation_name)
if not video_uri:
return False
# Step 3: Download video
success = self.download_video(video_uri, output_filename)
if success:
print("=" * 60)
print("🎉 SUCCESS! Video generation complete!")
print(f"📁 Video saved as: {output_filename}")
print("=" * 60)
else:
print("=" * 60)
print("❌ FAILED! Video generation or download failed")
print("=" * 60)
return success
def main():
"""
Example usage of the VeoVideoGenerator.
Configure these environment variables:
- LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
- LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
"""
# Configuration from environment or defaults
base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
print("🚀 Starting Veo Video Generation Example")
print(f"📡 Using LiteLLM proxy at: {base_url}")
# Initialize generator
generator = VeoVideoGenerator(base_url=base_url, api_key=api_key)
# Example prompts - try different ones!
example_prompts = [
"A cat playing with a ball of yarn in a sunny garden",
"Ocean waves crashing against rocky cliffs at sunset",
"A bustling city street with people walking and cars passing by",
"A peaceful forest with sunlight filtering through the trees"
]
# Use first example or get from user
prompt = example_prompts[0]
print(f"🎬 Using prompt: '{prompt}'")
# Generate and download video
success = generator.generate_and_download(prompt)
if success:
print("\n✅ Example completed successfully!")
print("💡 Try modifying the prompt in the script for different videos!")
else:
print("\n❌ Example failed!")
print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key")
# Troubleshooting tips
print("\n🔍 Troubleshooting:")
print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through")
print("2. Verify your Google AI Studio API key has Veo access")
print("3. Check that your prompt meets Veo's content guidelines")
print("4. Review the LiteLLM proxy logs for detailed error information")
if __name__ == "__main__":
main()

187
db_scripts/migrate_keys.py Normal file
View file

@ -0,0 +1,187 @@
from prisma import Prisma
import csv
import json
import asyncio
from datetime import datetime
from typing import Optional, List, Dict, Any
import os
## VARIABLES
DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/litellm"
CSV_FILE_PATH = "./path_to_csv.csv"
os.environ["DATABASE_URL"] = DATABASE_URL
async def parse_csv_value(value: str, field_type: str) -> Any:
"""Parse CSV values according to their expected types"""
if value == "NULL" or value == "" or value is None:
return None
if field_type == "boolean":
return value.lower() == "true"
elif field_type == "float":
return float(value)
elif field_type == "int":
return int(value) if value.isdigit() else None
elif field_type == "bigint":
return int(value) if value.isdigit() else None
elif field_type == "datetime":
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except:
return None
elif field_type == "json":
try:
return value if value else json.dumps({})
except:
return json.dumps({})
elif field_type == "string_array":
# Handle string arrays like {default-models}
if value.startswith("{") and value.endswith("}"):
content = value[1:-1] # Remove braces
if content:
return [item.strip() for item in content.split(",")]
else:
return []
return []
else:
return value
async def migrate_verification_tokens():
"""Main migration function"""
prisma = Prisma()
await prisma.connect()
try:
# Read CSV file
csv_file_path = CSV_FILE_PATH
with open(csv_file_path, "r", encoding="utf-8") as file:
csv_reader = csv.DictReader(file)
processed_count = 0
error_count = 0
for row in csv_reader:
try:
# Replace 'default-team' with the specified UUID
team_id = row.get("team_id")
if team_id == "NULL" or team_id == "":
team_id = None
# Prepare data for insertion
verification_token_data = {
"token": row["token"],
"key_name": await parse_csv_value(row["key_name"], "string"),
"key_alias": await parse_csv_value(row["key_alias"], "string"),
"soft_budget_cooldown": await parse_csv_value(
row["soft_budget_cooldown"], "boolean"
),
"spend": await parse_csv_value(row["spend"], "float"),
"expires": await parse_csv_value(row["expires"], "datetime"),
"models": await parse_csv_value(row["models"], "string_array"),
"aliases": await parse_csv_value(row["aliases"], "json"),
"config": await parse_csv_value(row["config"], "json"),
"user_id": await parse_csv_value(row["user_id"], "string"),
"team_id": team_id,
"permissions": await parse_csv_value(
row["permissions"], "json"
),
"max_parallel_requests": await parse_csv_value(
row["max_parallel_requests"], "int"
),
"metadata": await parse_csv_value(row["metadata"], "json"),
"tpm_limit": await parse_csv_value(row["tpm_limit"], "bigint"),
"rpm_limit": await parse_csv_value(row["rpm_limit"], "bigint"),
"max_budget": await parse_csv_value(row["max_budget"], "float"),
"budget_duration": await parse_csv_value(
row["budget_duration"], "string"
),
"budget_reset_at": await parse_csv_value(
row["budget_reset_at"], "datetime"
),
"allowed_cache_controls": await parse_csv_value(
row["allowed_cache_controls"], "string_array"
),
"model_spend": await parse_csv_value(
row["model_spend"], "json"
),
"model_max_budget": await parse_csv_value(
row["model_max_budget"], "json"
),
"budget_id": await parse_csv_value(row["budget_id"], "string"),
"blocked": await parse_csv_value(row["blocked"], "boolean"),
"created_at": await parse_csv_value(
row["created_at"], "datetime"
),
"updated_at": await parse_csv_value(
row["updated_at"], "datetime"
),
"allowed_routes": await parse_csv_value(
row["allowed_routes"], "string_array"
),
"object_permission_id": await parse_csv_value(
row["object_permission_id"], "string"
),
"created_by": await parse_csv_value(
row["created_by"], "string"
),
"updated_by": await parse_csv_value(
row["updated_by"], "string"
),
"organization_id": await parse_csv_value(
row["organization_id"], "string"
),
}
# Remove None values to use database defaults
verification_token_data = {
k: v
for k, v in verification_token_data.items()
if v is not None
}
# Check if token already exists
existing_token = await prisma.litellm_verificationtoken.find_unique(
where={"token": verification_token_data["token"]}
)
if existing_token:
print(
f"Token {verification_token_data['token']} already exists, skipping..."
)
continue
# Insert the record
await prisma.litellm_verificationtoken.create(
data=verification_token_data
)
processed_count += 1
print(
f"Successfully migrated token: {verification_token_data['token']}"
)
except Exception as e:
error_count += 1
print(
f"Error processing row with token {row.get('token', 'unknown')}: {str(e)}"
)
continue
print(f"\nMigration completed!")
print(f"Successfully processed: {processed_count} records")
print(f"Errors encountered: {error_count} records")
except Exception as e:
print(f"Migration failed: {str(e)}")
finally:
await prisma.disconnect()
if __name__ == "__main__":
asyncio.run(migrate_verification_tokens())

3
deploy/Dockerfile.ghcr_base
View file

@ -8,7 +8,8 @@ WORKDIR /app
COPY config.yaml .
# Make sure your docker/entrypoint.sh is executable
RUN chmod +x docker/entrypoint.sh
# Convert Windows line endings to Unix
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
# Expose the necessary port
EXPOSE 4000/tcp

6
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,13 +18,13 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.3.0
version: 1.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: v1.50.2
appVersion: v1.80.12
dependencies:
- name: "postgresql"
@ -33,5 +33,5 @@ dependencies:
condition: db.deployStandalone
- name: redis
version: ">=18.0.0"
repository: oci://registry-1.docker.io/bitnamicharts
repository: oci://registry-1.docker.io/bitnamicharts
condition: redis.enabled

135
deploy/charts/litellm-helm/README.md
View file

@ -10,33 +10,75 @@
- Helm 3.8.0+
If `db.deployStandalone` is used:
- PV provisioner support in the underlying infrastructure
If `db.useStackgresOperator` is used (not yet implemented):
- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing.
- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing.
## Parameters
### LiteLLM Proxy Deployment Settings
| Name | Description | Value |
| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` |
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key is generated. | N/A |
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` |
| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` |
| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` |
| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` |
| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` |
| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` |
| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` |
| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A |
| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | N/A |
| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | `[]` |
| Name | Description | Value |
| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- |
| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` |
| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A |
| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A |
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A |
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `image.repository` | LiteLLM Proxy image repository | `docker.litellm.ai/berriai/litellm` |
| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` |
| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` |
| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` |
| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` |
| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` |
| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` |
| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` |
| `ingress.labels` | Additional labels for the Ingress resource | `{}` |
| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A |
| `proxyConfigMap.create` | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it. | `true` |
| `proxyConfigMap.name` | When `create=false`, name of the existing ConfigMap to mount. | `""` |
| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` |
| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. |
| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` |
| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` |
| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` |
#### Example `environmentSecrets` Secret
#### Example `proxy_config` ConfigMap from values (default):
```
proxyConfigMap:
create: true
key: "config.yaml"
proxy_config:
general_settings:
master_key: os.environ/PROXY_MASTER_KEY
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
api_key: eXaMpLeOnLy
```
#### Example using existing `proxyConfigMap` instead of creating it:
```
proxyConfigMap:
create: false
name: my-litellm-config
key: config.yaml
# proxy_config is ignored in this mode
```
#### Example `environmentSecrets` Secret
```
apiVersion: v1
@ -49,21 +91,23 @@ type: Opaque
```
### Database Settings
| Name | Description | Value |
| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
| `db.useExisting` | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below. | `false` |
| `db.endpoint` | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to. | `localhost` |
| `db.database` | If `db.useExisting` is `true`, the name of the existing database to connect to. | `litellm` |
| `db.url` | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value. | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` |
| `db.secret.name` | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials. | `postgres` |
| `db.secret.usernameKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance. | `username` |
| `db.secret.passwordKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user. | `password` |
| `db.useStackgresOperator` | Not yet implemented. | `false` |
| `db.deployStandalone` | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups. | `true` |
| `postgresql.*` | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml) |
| `postgresql.auth.*` | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used. | `NoTaGrEaTpAsSwOrD` |
| Name | Description | Value |
| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ |
| `db.useExisting` | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below. | `false` |
| `db.endpoint` | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to. | `localhost` |
| `db.database` | If `db.useExisting` is `true`, the name of the existing database to connect to. | `litellm` |
| `db.url` | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value. | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` |
| `db.secret.name` | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials. | `postgres` |
| `db.secret.usernameKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance. | `username` |
| `db.secret.passwordKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user. | `password` |
| `db.useStackgresOperator` | Not yet implemented. | `false` |
| `db.deployStandalone` | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups. | `true` |
| `postgresql.*` | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml) |
| `postgresql.auth.*` | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used. | `NoTaGrEaTpAsSwOrD` |
#### Example Postgres `db.useExisting` Secret
```yaml
apiVersion: v1
kind: Secret
@ -101,29 +145,46 @@ metadata:
name: litellm-env-secret
type: Opaque
data:
SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded
SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded
ANOTHER_PASSWORD: AAZbUGVXeU5e0ZB # base64 encoded
```
Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472732157221e04c6b15e3b3f094)
### Migration Job Settings
The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments.
| Name | Description | Value |
| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------- |
| `migrationJob.enabled` | Enable or disable the schema migration Job | `true` |
| `migrationJob.backoffLimit` | Backoff limit for Job restarts | `4` |
| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs | `120` |
| `migrationJob.annotations` | Additional annotations for the migration job pod | `{}` |
| `migrationJob.extraContainers` | Additional containers to run alongside the migration job | `[]` |
| `migrationJob.hooks.argocd.enabled` | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy) | `true` |
| `migrationJob.hooks.helm.enabled` | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` |
| `migrationJob.hooks.helm.weight` | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified. | N/A |
## Accessing the Admin UI
When browsing to the URL published per the settings in `ingress.*`, you will
be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal
be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal
(from the `litellm` pod's perspective) URL published by the `<RELEASE>-litellm`
Kubernetes Service. If the deployment uses the default settings for this
Kubernetes Service. If the deployment uses the default settings for this
service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`.
The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
was not provided to the helm command line, the `masterkey` is a randomly
generated string stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
generated string in the `sk-...` format stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
```bash
kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.masterkey}"
```
## Admin UI Limitations
At the time of writing, the Admin UI is unable to add models. This is because
At the time of writing, the Admin UI is unable to add models. This is because
it would need to update the `config.yaml` file which is a exposed ConfigMap, and
therefore, read-only. This is a limitation of this helm chart, not the Admin UI
therefore, read-only. This is a limitation of this helm chart, not the Admin UI
itself.

1
deploy/charts/litellm-helm/templates/NOTES.txt
View file

@ -20,3 +20,4 @@
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.*

4
deploy/charts/litellm-helm/templates/configmap-litellm.yaml
View file

@ -1,7 +1,9 @@
{{- if .Values.proxyConfigMap.create }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "litellm.fullname" . }}-config
data:
config.yaml: |
{{ .Values.proxy_config | toYaml | indent 6 }}
{{ .Values.proxy_config | toYaml | indent 6 }}
{{- end }}

74
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -1,11 +1,16 @@
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
{{- toYaml .Values.deploymentAnnotations | nindent 4 }}
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- if .Values.deploymentLabels }}
{{- toYaml .Values.deploymentLabels | nindent 4 }}
{{- end }}
spec:
{{- if not .Values.autoscaling.enabled }}
{{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
replicas: {{ .Values.replicaCount }}
{{- end }}
selector:
@ -14,7 +19,9 @@ spec:
template:
metadata:
annotations:
{{- if .Values.proxyConfigMap.create }}
checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
@ -31,6 +38,10 @@ spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- with .Values.extraInitContainers }}
initContainers:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: {{ include "litellm.name" . }}
securityContext:
@ -69,7 +80,14 @@ spec:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.passwordKey }}
- name: DATABASE_HOST
{{- if .Values.db.secret.endpointKey }}
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.endpointKey }}
{{- else }}
value: {{ .Values.db.endpoint }}
{{- end }}
- name: DATABASE_NAME
value: {{ .Values.db.database }}
- name: DATABASE_URL
@ -78,8 +96,8 @@ spec:
- name: PROXY_MASTER_KEY
valueFrom:
secretKeyRef:
name: {{ include "litellm.fullname" . }}-masterkey
key: masterkey
name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }}
key: {{ .Values.masterkeySecretKey | default "masterkey" }}
{{- if .Values.redis.enabled }}
- name: REDIS_HOST
value: {{ include "litellm.redis.serviceName" . }}
@ -97,6 +115,15 @@ spec:
value: {{ $val | quote }}
{{- end }}
{{- end }}
{{- if .Values.separateHealthApp }}
- name: SEPARATE_HEALTH_APP
value: "1"
- name: SEPARATE_HEALTH_PORT
value: {{ .Values.separateHealthPort | default "8081" | quote }}
{{- end }}
{{- with .Values.extraEnvVars }}
{{- toYaml . | nindent 12 }}
{{- end }}
envFrom:
{{- range .Values.environmentSecrets }}
- secretRef:
@ -106,33 +133,49 @@ spec:
- configMapRef:
name: {{ . }}
{{- end }}
{{- if .Values.command }}
command: {{ toYaml .Values.command | nindent 12 }}
{{- end }}
{{- if .Values.args }}
args: {{ toYaml .Values.args | nindent 12 }}
{{- else }}
args:
- --config
- /etc/litellm/config.yaml
{{ if .Values.numWorkers }}
- --num_workers
- {{ .Values.numWorkers | quote }}
{{- end }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
{{- if .Values.separateHealthApp }}
- name: health
containerPort: {{ .Values.separateHealthPort | default 8081 }}
protocol: TCP
{{- end }}
livenessProbe:
httpGet:
path: /health/liveliness
port: http
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
readinessProbe:
httpGet:
path: /health/readiness
port: http
# Give the container time to start up. Up to 5 minutes (10 * 30 seconds)
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
startupProbe:
httpGet:
path: /health/readiness
port: http
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
failureThreshold: 30
periodSeconds: 10
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: litellm-config
mountPath: /etc/litellm/
mountPath: /etc/litellm/config.yaml
subPath: config.yaml
{{ if .Values.securityContext.readOnlyRootFilesystem }}
- name: tmp
mountPath: /tmp
@ -144,6 +187,10 @@ spec:
{{- with .Values.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.extraContainers }}
{{- toYaml . | nindent 8 }}
{{- end }}
@ -161,9 +208,13 @@ spec:
{{- end }}
- name: litellm-config
configMap:
{{- if .Values.proxyConfigMap.create }}
name: {{ include "litellm.fullname" . }}-config
{{- else }}
name: {{ .Values.proxyConfigMap.name }}
{{- end }}
items:
- key: "config.yaml"
- key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
path: "config.yaml"
{{- with .Values.volumes }}
{{- toYaml . | nindent 8 }}
@ -180,3 +231,8 @@ spec:
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | default 90 }}
{{- if .Values.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
{{- end }}

6
deploy/charts/litellm-helm/templates/extra-resources.yaml Normal file
View file

@ -0,0 +1,6 @@
{{- if .Values.extraResources }}
{{- range .Values.extraResources }}
---
{{ toYaml . | nindent 0 }}
{{- end }}
{{- end }}

Some files were not shown because too many files have changed in this diff Show more