scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

fix: resolve merge conflicts with main and fix broken doc link

Browse source 
- Fixed broken link /docs/anthropic_unified/ -> /docs/providers/anthropic
- Resolved merge conflict in sidebars.js keeping main's new structure
- Added tutorials/openclaw to the tutorials list
This commit is contained in:
Shin 2026-02-07 08:44:20 +00:00
commit 64ecc9ba9e
6442 changed files with 1635890 additions and 138347 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4366
.circleci/config.yml
View file

File diff suppressed because it is too large Load diff

18
.circleci/requirements.txt
View file

@ -1,12 +1,20 @@
# used by CI/CD testing
openai
openai==1.100.1
python-dotenv
tiktoken
importlib_metadata
cohere
redis
redis==5.2.1
redisvl==0.4.1
anthropic
orjson
pydantic==1.10.14
orjson==3.10.12 # fast /embedding responses
pydantic==2.11.0
google-cloud-aiplatform==1.43.0
redisvl==0.0.7 # semantic caching
google-cloud-iam==2.19.1
fastapi-sso==0.16.0
uvloop==0.21.0
mcp==1.25.0 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
fastuuid==0.12.0
responses==0.25.7 # for proxy client tests
pytest-retry==1.6.3 # for automatic test retries

57
.devcontainer/devcontainer.json Normal file
View file

@ -0,0 +1,57 @@
{
"name": "Python 3.11",
// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
"image": "mcr.microsoft.com/devcontainers/python:3.11-bookworm",
// https://github.com/devcontainers/images/tree/main/src/python
// https://mcr.microsoft.com/en-us/product/devcontainers/python/tags
// "build": {
// "dockerfile": "Dockerfile",
// "context": ".."
// },
// Features to add to the dev container. More info: https://containers.dev/features.
"features": {
"ghcr.io/devcontainers/features/node:1": {
"version": "lts"
},
"ghcr.io/devcontainers/features/docker-in-docker:2": {}
},
// Configure tool-specific properties.
"customizations": {
// Configure properties specific to VS Code.
"vscode": {
"settings": {},
"extensions": [
"ms-python.python",
"ms-python.vscode-pylance",
"GitHub.copilot",
"GitHub.copilot-chat",
"ms-python.autopep8"
]
}
},
// Use 'forwardPorts' to make a list of ports inside the container available locally.
"forwardPorts": [4000],
"containerEnv": {
"LITELLM_LOG": "DEBUG"
},
// Use 'portsAttributes' to set default properties for specific forwarded ports.
// More info: https://containers.dev/implementors/json_reference/#port-attributes
"portsAttributes": {
"4000": {
"label": "LiteLLM Server",
"onAutoForward": "notify"
}
},
// More info: https://aka.ms/dev-containers-non-root.
// "remoteUser": "litellm",
// Use 'postCreateCommand' to run commands after the container is created.
"postCreateCommand": "bash ./.devcontainer/post-create.sh"
}

17
.devcontainer/post-create.sh Normal file
View file

@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -e
echo "[post-create] Installing poetry via pip"
python -m pip install --upgrade pip
python -m pip install poetry
echo "[post-create] Installing Python dependencies (poetry)"
poetry install --with dev --extras proxy
echo "[post-create] Generating Prisma client"
poetry run prisma generate
echo "[post-create] Installing npm dependencies"
cd ui/litellm-dashboard && npm install --no-audit --no-fund
echo "[post-create] Done"

49
.dockerignore
View file

@ -3,3 +3,52 @@ cookbook
.circleci
.github
tests
.git
.devcontainer
*.tgz
log.txt
docker/Dockerfile.*
# Claude Flow generated files (must be excluded from Docker build)
.claude/
.claude-flow/
.swarm/
.hive-mind/
memory/
coordination/
claude-flow
.mcp.json
hive-mind-prompt-*.txt
# Python virtual environments and version managers
.venv/
venv/
**/.venv/
**/venv/
.python-version
.pyenv/
__pycache__/
**/__pycache__/
*.pyc
.mypy_cache/
.pytest_cache/
.ruff_cache/
**/pyvenv.cfg
# Common project exclusions
.vscode
*.pyo
*.pyd
.Python
env/
.pytest_cache
.coverage
htmlcov/
dist/
build/
*.egg-info/
.DS_Store
node_modules/
*.log
.env
.env.local

11
.env.example
View file

@ -1,6 +1,6 @@
# OpenAI
OPENAI_API_KEY = ""
OPENAI_API_BASE = ""
OPENAI_BASE_URL = ""
# Cohere
COHERE_API_KEY = ""
# OpenRouter
@ -20,3 +20,12 @@ REPLICATE_API_TOKEN = ""
ANTHROPIC_API_KEY = ""
# Infisical
INFISICAL_TOKEN = ""
# Novita AI
NOVITA_API_KEY = ""
# INFINITY
INFINITY_API_KEY = ""
# Development Configs
LITELLM_MASTER_KEY = "sk-1234"
DATABASE_URL = "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
STORE_MODEL_IN_DB = "True"

10
.git-blame-ignore-revs Normal file
View file

@ -0,0 +1,10 @@
# Add the commit hash of any commit you want to ignore in `git blame` here.
# One commit hash per line.
#
# The GitHub Blame UI will use this file automatically!
#
# Run this command to always ignore formatting commits in `git blame`
# git config blame.ignoreRevsFile .git-blame-ignore-revs
# Update pydantic code to fix warnings (GH-3600)
876840e9957bc7e9f7d6a2b58c4d7c53dad16481

111
.gitguardian.yaml Normal file
View file

@ -0,0 +1,111 @@
version: 2
secret:
# Exclude files and paths by globbing
ignored_paths:
- "**/*.whl"
- "**/*.pyc"
- "**/__pycache__/**"
- "**/node_modules/**"
- "**/dist/**"
- "**/build/**"
- "**/.git/**"
- "**/venv/**"
- "**/.venv/**"
# Large data/metadata files that don't need scanning
- "**/model_prices_and_context_window*.json"
- "**/*_metadata/*.txt"
- "**/tokenizers/*.json"
- "**/tokenizers/*"
- "miniconda.sh"
# Build outputs and static assets
- "litellm/proxy/_experimental/out/**"
- "ui/litellm-dashboard/public/**"
- "**/swagger/*.js"
- "**/*.woff"
- "**/*.woff2"
- "**/*.avif"
- "**/*.webp"
# Test data files
- "**/tests/**/data_map.txt"
- "tests/**/*.txt"
# Documentation and other non-code files
- "docs/**"
- "**/*.md"
- "**/*.lock"
- "poetry.lock"
- "package-lock.json"
# Ignore security incidents with the SHA256 of the occurrence (false positives)
ignored_matches:
# === Current detected false positives (SHA-based) ===
# gcs_pub_sub_body - folder name, not a password
- name: GCS pub/sub test folder name
match: 75f377c456eede69e5f6e47399ccee6016a2a93cc5dd11db09cc5b1359ae569a
# os.environ/APORIA_API_KEY_1 - environment variable reference
- name: Environment variable reference APORIA_API_KEY_1
match: e2ddeb8b88eca97a402559a2be2117764e11c074d86159ef9ad2375dea188094
# os.environ/APORIA_API_KEY_2 - environment variable reference
- name: Environment variable reference APORIA_API_KEY_2
match: 09aa39a29e050b86603aa55138af1ff08fb86a4582aa965c1bd0672e1575e052
# oidc/circleci_v2/ - test authentication path, not a secret
- name: OIDC CircleCI test path
match: feb3475e1f89a65b7b7815ac4ec597e18a9ec1847742ad445c36ca617b536e15
# text-davinci-003 - OpenAI model identifier, not a secret
- name: OpenAI model identifier text-davinci-003
match: c489000cf6c7600cee0eefb80ad0965f82921cfb47ece880930eb7e7635cf1f1
# Base64 Basic Auth in test_pass_through_endpoints.py - test fixture, not a real secret
- name: Test Base64 Basic Auth header in pass_through_endpoints test
match: 61bac0491f395040617df7ef6d06029eac4d92a4457ac784978db80d97be1ae0
# PostgreSQL password "postgres" in CI configs - standard test database password
- name: Test PostgreSQL password in CI configurations
match: 6e0d657eb1f0fbc40cf0b8f3c3873ef627cc9cb7c4108d1c07d979c04bc8a4bb
# Bearer token in locustfile.py - test/example API key for load testing
- name: Test Bearer token in locustfile load test
match: 2a0abc2b0c3c1760a51ffcdf8d6b1d384cef69af740504b1cfa82dd70cdc7ff9
# Inkeep API key in docusaurus.config.js - public documentation site key
- name: Inkeep API key in documentation config
match: c366657791bfb5fc69045ec11d49452f09a0aebbc8648f94e2469b4025e29a75
# Langfuse credentials in test_completion.py - test credentials for integration test
- name: Langfuse test credentials in test_completion
match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
# Test password "sk-1234" in e2e test fixtures - test fixture, not a real secret
- name: Test password in e2e test fixtures
match: ce32b547202e209ec1dd50107b64be4cfcf2eb15c3b4f8e9dc611ef747af634f
# === Preventive patterns for test keys (pattern-based) ===
# Test API keys (124 instances across 45 files)
- name: Test API keys with sk-test prefix
match: sk-test-
# Mock API keys
- name: Mock API keys with sk-mock prefix
match: sk-mock-
# Fake API keys
- name: Fake API keys with sk-fake prefix
match: sk-fake-
# Generic test API key patterns
- name: Test API key patterns
match: test-api-key
- name: Short fake sk keys (1–9 digits only)
match: \bsk-\d{1,9}\b

45
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View file

@ -7,6 +7,16 @@ body:
attributes:
value: |
Thanks for taking the time to fill out this bug report!
**💡 Tip:** See our [Troubleshooting Guide](https://docs.litellm.ai/docs/troubleshoot) for what information to include.
- type: checkboxes
id: duplicate-check
attributes:
label: Check for existing issues
description: Please search to see if an issue already exists for the bug you encountered.
options:
- label: I have searched the existing issues and checked that my issue is not a duplicate.
required: true
- type: textarea
id: what-happened
attributes:
@ -16,12 +26,47 @@ body:
value: "A bug happened!"
validations:
required: true
- type: textarea
id: steps-to-reproduce
attributes:
label: Steps to Reproduce
description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
placeholder: |
1. config.yaml file/ .env file/ etc.
2. Run the following code...
3. Observe the error...
value: |
1.
2.
3.
validations:
required: true
- type: textarea
id: logs
attributes:
label: Relevant log output
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
render: shell
- type: dropdown
id: component
attributes:
label: What part of LiteLLM is this about?
options:
- ''
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
- "Docs"
- "Other"
validations:
required: true
- type: input
id: version
attributes:
label: What LiteLLM version are you on ?
placeholder: v1.53.1
validations:
required: true
- type: input
id: contact
attributes:

31
.github/ISSUE_TEMPLATE/feature_request.yml vendored
View file

@ -7,6 +7,14 @@ body:
attributes:
value: |
Thanks for making LiteLLM better!
- type: checkboxes
id: duplicate-check
attributes:
label: Check for existing issues
description: Please search to see if an issue already exists for the feature you are requesting.
options:
- label: I have searched the existing issues and checked that my issue is not a duplicate.
required: true
- type: textarea
id: the-feature
attributes:
@ -22,6 +30,29 @@ body:
description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too.
validations:
required: true
- type: dropdown
id: component
attributes:
label: What part of LiteLLM is this about?
options:
- ''
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
- "Docs"
- "Other"
validations:
required: true
- type: dropdown
id: hiring-interest
attributes:
label: LiteLLM is hiring a founding backend engineer, are you interested in joining us and shipping to all our users?
description: If yes, apply here - https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer
options:
- "No"
- "Yes"
validations:
required: true
- type: input
id: contact
attributes:

10
.github/dependabot.yaml vendored Normal file
View file

@ -0,0 +1,10 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "daily"
groups:
github-actions:
patterns:
- "*"

42
.github/pull_request_template.md vendored Normal file
View file

@ -0,0 +1,42 @@
## Relevant issues
<!-- e.g. "Fixes #000" -->
## Pre-Submission checklist
**Please complete all items before asking a LiteLLM maintainer to review your PR**
- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
## CI (LiteLLM team)
> **CI status guideline:**
>
> - 50-55 passing tests: main is stable with minor issues.
> - 45-49 passing tests: acceptable but needs attention
> - <= 40 passing tests: unstable; be careful with your merges and assess the risk.
- [ ] **Branch creation CI run**
Link:
- [ ] **CI run for the last commit**
Link:
- [ ] **Merge / cherry-pick CI run**
Links:
## Type
<!-- Select the type of Pull Request -->
<!-- Keep only the necessary ones -->
🆕 New Feature
🐛 Bug Fix
🧹 Refactoring
📖 Documentation
🚄 Infrastructure
✅ Test
## Changes

133
.github/scripts/scan_keywords.py vendored Normal file
View file

@ -0,0 +1,133 @@
#!/usr/bin/env python3
import json
import os
import sys
import urllib.request
import urllib.error
def read_event_payload() -> dict:
event_path = os.environ.get("GITHUB_EVENT_PATH")
if not event_path or not os.path.exists(event_path):
return {}
with open(event_path, "r", encoding="utf-8") as f:
return json.load(f)
def get_issue_text(event: dict) -> tuple[str, str, int, str, str]:
issue = event.get("issue") or {}
title = (issue.get("title") or "").strip()
body = (issue.get("body") or "").strip()
number = issue.get("number") or 0
html_url = issue.get("html_url") or ""
author = ((issue.get("user") or {}).get("login") or "").strip()
return title, body, number, html_url, author
def detect_keywords(text: str, keywords: list[str]) -> list[str]:
lowered = text.lower()
matches = []
for keyword in keywords:
k = keyword.strip().lower()
if not k:
continue
if k in lowered:
matches.append(keyword.strip())
# Deduplicate while preserving order
seen = set()
unique_matches = []
for m in matches:
if m not in seen:
unique_matches.append(m)
seen.add(m)
return unique_matches
def send_webhook(webhook_url: str, payload: dict) -> None:
if not webhook_url:
return
data = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(
webhook_url,
data=data,
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with urllib.request.urlopen(req, timeout=10) as resp:
resp.read()
except urllib.error.HTTPError as e:
print(f"Webhook HTTP error: {e.code} {e.reason}", file=sys.stderr)
except urllib.error.URLError as e:
print(f"Webhook URL error: {e.reason}", file=sys.stderr)
except Exception as e:
print(f"Webhook unexpected error: {e}", file=sys.stderr)
def _excerpt(text: str, max_len: int = 400) -> str:
if not text:
return ""
# Keep original formatting
if len(text) <= max_len:
return text
return text[: max_len - 1] + "…"
def main() -> int:
event = read_event_payload()
if not event:
print("::warning::No event payload found; exiting without labeling.")
return 0
# Read issue details
title, body, number, html_url, author = get_issue_text(event)
combined_text = f"{title}\n\n{body}".strip()
# Keywords from env or defaults
keywords_env = os.environ.get("KEYWORDS", "")
default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
matches = detect_keywords(combined_text, keywords)
found = bool(matches)
# Emit outputs
github_output = os.environ.get("GITHUB_OUTPUT")
if github_output:
with open(github_output, "a", encoding="utf-8") as fh:
fh.write(f"found={'true' if found else 'false'}\n")
fh.write(f"matches={','.join(matches)}\n")
# Optional webhook notification
webhook_url = os.environ.get("PROVIDER_ISSUE_WEBHOOK_URL", "").strip()
if found and webhook_url:
repo_full = (event.get("repository") or {}).get("full_name", "")
title_part = f"*{title}*" if title else "New issue"
author_part = f" by @{author}" if author else ""
body_preview = _excerpt(body)
preview_block = f"\n{body_preview}" if body_preview else ""
payload = {
"text": (
f"New issue 🚨\n"
f"{title_part}\n\n{preview_block}\n"
f"<{html_url}|View issue>\n"
f"Author: {author}"
)
}
send_webhook(webhook_url, payload)
# Print a short log line for Actions UI
if found:
print(f"Detected provider keywords: {', '.join(matches)}")
else:
print("No provider keywords detected.")
return 0
if __name__ == "__main__":
raise SystemExit(main())

35
.github/workflows/README.md vendored Normal file
View file

@ -0,0 +1,35 @@
# Simple PyPI Publishing
A GitHub workflow to manually publish LiteLLM packages to PyPI with a specified version.
## How to Use
1. Go to the **Actions** tab in the GitHub repository
2. Select **Simple PyPI Publish** from the workflow list
3. Click **Run workflow**
4. Enter the version to publish (e.g., `1.74.10`)
## What the Workflow Does
1. **Updates** the version in `pyproject.toml`
2. **Copies** the model prices backup file
3. **Builds** the Python package
4. **Publishes** to PyPI
## Prerequisites
Make sure the following secret is configured in the repository:
- `PYPI_PUBLISH_PASSWORD`: PyPI API token for authentication
## Example Usage
- Version: `1.74.11` → Publishes as v1.74.11
- Version: `1.74.10-hotfix1` → Publishes as v1.74.10-hotfix1
## Features
- ✅ Manual trigger with version input
- ✅ Automatic version updates in `pyproject.toml`
- ✅ Repository safety check (only runs on official repo)
- ✅ Clean package building and publishing
- ✅ Success confirmation with PyPI package link

28
.github/workflows/auto_update_price_and_context_window.yml vendored Normal file
View file

@ -0,0 +1,28 @@
name: Updates model_prices_and_context_window.json and Create Pull Request
on:
schedule:
- cron: "0 0 * * 0" # Run every Sundays at midnight
#- cron: "0 0 * * *" # Run daily at midnight
jobs:
auto_update_price_and_context_window:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Install Dependencies
run: |
pip install aiohttp
- name: Update JSON Data
run: |
python ".github/workflows/auto_update_price_and_context_window_file.py"
- name: Create Pull Request
run: |
git add model_prices_and_context_window.json
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
gh pr create --title "Update model_prices_and_context_window.json file" \
--body "Automated update for model_prices_and_context_window.json" \
--head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \
--base main
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}

159
.github/workflows/auto_update_price_and_context_window_file.py vendored Normal file
View file

@ -0,0 +1,159 @@
import asyncio
import aiohttp
import json
# Asynchronously fetch data from a given URL
async def fetch_data(url):
try:
# Create an asynchronous session
async with aiohttp.ClientSession() as session:
# Send a GET request to the URL
async with session.get(url) as resp:
# Raise an error if the response status is not OK
resp.raise_for_status()
# Parse the response JSON
resp_json = await resp.json()
print("Fetch the data from URL.")
# Return the 'data' field from the JSON response
return resp_json['data']
except Exception as e:
# Print an error message if fetching data fails
print("Error fetching data from URL:", e)
return None
# Synchronize local data with remote data
def sync_local_data_with_remote(local_data, remote_data):
# Update existing keys in local_data with values from remote_data
for key in (set(local_data) & set(remote_data)):
local_data[key].update(remote_data[key])
# Add new keys from remote_data to local_data
for key in (set(remote_data) - set(local_data)):
local_data[key] = remote_data[key]
# Write data to the json file
def write_to_file(file_path, data):
try:
# Open the file in write mode
with open(file_path, "w") as file:
# Dump the data as JSON into the file
json.dump(data, file, indent=4)
print("Values updated successfully.")
except Exception as e:
# Print an error message if writing to file fails
print("Error updating JSON file:", e)
# Update the existing models and add the missing models for OpenRouter
def transform_openrouter_data(data):
transformed = {}
for row in data:
# Add the fields 'max_tokens' and 'input_cost_per_token'
obj = {
"max_tokens": row["context_length"],
"input_cost_per_token": float(row["pricing"]["prompt"]),
}
# Add 'max_output_tokens' as a field if it is not None
if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None:
obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"])
# Add the field 'output_cost_per_token'
obj.update({
"output_cost_per_token": float(row["pricing"]["completion"]),
})
# Add field 'input_cost_per_image' if it exists and is non-zero
if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0:
obj['input_cost_per_image'] = float(row["pricing"]["image"])
# Add the fields 'litellm_provider' and 'mode'
obj.update({
"litellm_provider": "openrouter",
"mode": "chat"
})
# Add the 'supports_vision' field if the modality is 'multimodal'
if row.get('architecture', {}).get('modality') == 'multimodal':
obj['supports_vision'] = True
# Use a composite key to store the transformed object
transformed[f'openrouter/{row["id"]}'] = obj
return transformed
# Update the existing models and add the missing models for Vercel AI Gateway
def transform_vercel_ai_gateway_data(data):
transformed = {}
for row in data:
obj = {
"max_tokens": row["context_window"],
"input_cost_per_token": float(row["pricing"]["input"]),
"output_cost_per_token": float(row["pricing"]["output"]),
'max_output_tokens': row['max_tokens'],
'max_input_tokens': row["context_window"],
}
# Handle cache pricing if available
if "pricing" in row:
if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
mode = "embedding" if "embedding" in row["id"].lower() else "chat"
obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
return transformed
# Load local data from a specified file
def load_local_data(file_path):
try:
# Open the file in read mode
with open(file_path, "r") as file:
# Load and return the JSON data
return json.load(file)
except FileNotFoundError:
# Print an error message if the file is not found
print("File not found:", file_path)
return None
except json.JSONDecodeError as e:
# Print an error message if JSON decoding fails
print("Error decoding JSON:", e)
return None
def main():
local_file_path = "model_prices_and_context_window.json" # Path to the local data file
openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data
vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data
# Load local data from file
local_data = load_local_data(local_file_path)
# Fetch OpenRouter data
openrouter_data = asyncio.run(fetch_data(openrouter_url))
# Transform the fetched OpenRouter data
openrouter_data = transform_openrouter_data(openrouter_data)
# Fetch Vercel AI Gateway data
vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
# Transform the fetched Vercel AI Gateway data
vercel_data = transform_vercel_ai_gateway_data(vercel_data)
# Combine both datasets
all_remote_data = {**openrouter_data, **vercel_data}
# If both local and openrouter data are available, synchronize and save
if local_data and all_remote_data:
sync_local_data_with_remote(local_data, all_remote_data)
write_to_file(local_file_path, local_data)
else:
print("Failed to fetch model data from either local file or URL.")
# Entry point of the script
if __name__ == "__main__":
main()

29
.github/workflows/check_duplicate_issues.yml vendored Normal file
View file

@ -0,0 +1,29 @@
name: Check Duplicate Issues
on:
issues:
types: [opened, edited]
jobs:
check-duplicate:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Check for potential duplicates
uses: wow-actions/potential-duplicates@v1
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
label: potential-duplicate
threshold: 0.6
reaction: eyes
comment: |
**⚠️ Potential duplicate detected**
This issue appears similar to existing issue(s):
{{#issues}}
- [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
{{/issues}}
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.

43
.github/workflows/create_daily_staging_branch.yml vendored Normal file
View file

@ -0,0 +1,43 @@
name: Create Daily Staging Branch
on:
schedule:
- cron: '0 0,12 * * *' # Runs every 12 hours at midnight and noon UTC
workflow_dispatch: # Allow manual trigger
jobs:
create-staging-branch:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Create daily staging branch
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Configure Git user
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
# Generate branch name with MM_DD_YYYY format
BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')"
echo "Creating branch: $BRANCH_NAME"
# Fetch all branches
git fetch --all
# Check if the branch already exists
if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
echo "Branch $BRANCH_NAME already exists. Skipping creation."
else
echo "Creating new branch: $BRANCH_NAME"
# Create the new branch from main
git checkout -b $BRANCH_NAME origin/main
# Push the new branch
git push origin $BRANCH_NAME
echo "Successfully created and pushed branch: $BRANCH_NAME"
fi

215
.github/workflows/ghcr_deploy.yml vendored
View file

@ -5,8 +5,9 @@ on:
inputs:
tag:
description: "The tag version you want to build"
required: true
release_type:
description: "The release type you want to build. Can be 'latest', 'stable', 'dev'"
description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
type: string
default: "latest"
commit_hash:
@ -21,10 +22,23 @@ env:
# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
jobs:
# print commit hash, tag, and release type
print:
runs-on: ubuntu-latest
steps:
- run: |
echo "Commit hash: ${{ github.event.inputs.commit_hash }}"
echo "Tag: ${{ github.event.inputs.tag }}"
echo "Release type: ${{ github.event.inputs.release_type }}"
docker-hub-deploy:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
-
name: Set up QEMU
uses: docker/setup-qemu-action@v3
@ -41,33 +55,44 @@ jobs:
name: Build and push
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }}
-
name: Build and push litellm-database image
uses: docker/build-push-action@v5
with:
context: .
push: true
file: Dockerfile.database
file: ./docker/Dockerfile.database
tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }}
-
name: Build and push litellm-spend-logs image
uses: docker/build-push-action@v5
with:
context: .
push: true
file: ./litellm-js/spend-logs/Dockerfile
tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }}
-
name: Build and push litellm-non_root image
uses: docker/build-push-action@v5
with:
context: .
push: true
file: ./docker/Dockerfile.non_root
tags: litellm/litellm-non_root:${{ github.event.inputs.tag || 'latest' }}
build-and-push-image:
runs-on: ubuntu-latest
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
permissions:
contents: read
packages: write
#
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
@ -92,12 +117,60 @@ jobs:
- name: Build and push Docker image
uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8
with:
context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}}
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest'
tags: |
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }},
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
labels: ${{ steps.meta.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-image-ee:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for EE Dockerfile
id: meta-ee
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
# Configure multi platform Docker builds
- name: Set up QEMU
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
- name: Build and push EE Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: .
file: Dockerfile
push: true
tags: |
${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-ee.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-image-database:
runs-on: ubuntu-latest
permissions:
@ -106,6 +179,8 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
@ -128,11 +203,58 @@ jobs:
- name: Build and push Database Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}}
file: Dockerfile.database
context: .
file: ./docker/Dockerfile.database
push: true
tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
labels: ${{ steps.meta-database.outputs.labels }}
tags: |
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-database.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-image-non_root:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for non_root Dockerfile
id: meta-non_root
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-non_root
# Configure multi platform Docker builds
- name: Set up QEMU
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
- name: Build and push non_root Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: .
file: ./docker/Dockerfile.non_root
push: true
tags: |
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-non_root.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-image-spend-logs:
@ -143,6 +265,8 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
@ -165,17 +289,25 @@ jobs:
- name: Build and push Database Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}}
context: .
file: ./litellm-js/spend-logs/Dockerfile
push: true
tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
tags: |
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-helm-chart:
if: github.event.inputs.release_type != 'dev'
needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
@ -187,37 +319,38 @@ jobs:
- name: lowercase github.repository_owner
run: |
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
- name: Get LiteLLM Latest Tag
id: current_app_tag
uses: WyriHaximus/github-action-get-previous-tag@v1.3.0
- name: Get last published chart version
id: current_version
# Sync Helm chart version with LiteLLM release version (1-1 versioning)
# This allows users to easily map Helm chart versions to LiteLLM versions
# See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/
- name: Calculate chart and app versions
id: chart_version
shell: bash
run: |
CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
if [ -z "${CHART_LIST}" ]; then
echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
else
printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
fi
env:
HELM_EXPERIMENTAL_OCI: '1'
INPUT_TAG="${{ github.event.inputs.tag }}"
RELEASE_TYPE="${{ github.event.inputs.release_type }}"
# Automatically update the helm chart version one "patch" level
- name: Bump release version
id: bump_version
uses: christian-draeger/increment-semantic-version@1.1.0
with:
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
version-fragment: 'bug'
# Chart version = LiteLLM version without 'v' prefix (Helm semver convention)
# v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1
CHART_VERSION="${INPUT_TAG#v}"
# Add suffix for 'latest' releases (rc already has suffix in tag)
if [ "$RELEASE_TYPE" = "latest" ]; then
CHART_VERSION="${CHART_VERSION}-latest"
fi
# App version = Docker tag (keeps 'v' prefix to match Docker image tags)
APP_VERSION="${INPUT_TAG}"
echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
- uses: ./.github/actions/helm-oci-chart-releaser
with:
name: ${{ env.CHART_NAME }}
repository: ${{ env.REPO_OWNER }}
tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
tag: ${{ steps.chart_version.outputs.version }}
app_version: ${{ steps.chart_version.outputs.app_version }}
path: deploy/charts/${{ env.CHART_NAME }}
registry: ${{ env.REGISTRY }}
registry_username: ${{ github.actor }}
@ -273,7 +406,8 @@ jobs:
repo: context.repo.repo,
release_id: process.env.RELEASE_ID,
});
return response.data.body;
const formattedBody = JSON.stringify(response.data.body).slice(1, -1);
return formattedBody;
} catch (error) {
core.setFailed(error.message);
}
@ -286,14 +420,15 @@ jobs:
RELEASE_NOTES: ${{ steps.release-notes.outputs.result }}
run: |
curl -H "Content-Type: application/json" -X POST -d '{
"content": "New LiteLLM release ${{ env.RELEASE_TAG }}",
"content": "New LiteLLM release '"${RELEASE_TAG}"'",
"username": "Release Changelog",
"avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png",
"embeds": [
{
"title": "Changelog for LiteLLM ${{ env.RELEASE_TAG }}",
"description": "${{ env.RELEASE_NOTES }}",
"title": "Changelog for LiteLLM '"${RELEASE_TAG}"'",
"description": "'"${RELEASE_NOTES}"'",
"color": 2105893
}
]
}' $WEBHOOK_URL

47
.github/workflows/ghcr_helm_deploy.yml vendored
View file

@ -1,10 +1,12 @@
# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
# Standalone workflow to publish LiteLLM Helm Chart
# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release
name: Build, Publish LiteLLM Helm Chart. New Release
on:
workflow_dispatch:
inputs:
chartVersion:
description: "Update the helm chart's version to this"
tag:
description: "LiteLLM version tag (e.g., v1.81.0)"
required: true
# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
@ -31,34 +33,35 @@ jobs:
run: |
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
- name: Get LiteLLM Latest Tag
id: current_app_tag
uses: WyriHaximus/github-action-get-previous-tag@v1.3.0
- name: Get last published chart version
id: current_version
# Sync Helm chart version with LiteLLM release version (1-1 versioning)
- name: Calculate chart and app versions
id: chart_version
shell: bash
run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
env:
HELM_EXPERIMENTAL_OCI: '1'
run: |
INPUT_TAG="${{ github.event.inputs.tag }}"
# Automatically update the helm chart version one "patch" level
- name: Bump release version
id: bump_version
uses: christian-draeger/increment-semantic-version@1.1.0
with:
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
version-fragment: 'bug'
# Chart version = LiteLLM version without 'v' prefix
# v1.81.0 -> 1.81.0
CHART_VERSION="${INPUT_TAG#v}"
# App version = Docker tag (keeps 'v' prefix)
APP_VERSION="${INPUT_TAG}"
echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
- name: Lint helm chart
run: helm lint deploy/charts/litellm-helm
- uses: ./.github/actions/helm-oci-chart-releaser
with:
name: litellm-helm
repository: ${{ env.REPO_OWNER }}
tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
tag: ${{ steps.chart_version.outputs.version }}
app_version: ${{ steps.chart_version.outputs.app_version }}
path: deploy/charts/litellm-helm
registry: ${{ env.REGISTRY }}
registry_username: ${{ github.actor }}
registry_password: ${{ secrets.GITHUB_TOKEN }}
update_dependencies: true

27
.github/workflows/helm_unit_test.yml vendored Normal file
View file

@ -0,0 +1,27 @@
name: Helm unit test
on:
pull_request:
push:
branches:
- main
jobs:
unit-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Set up Helm 3.11.1
uses: azure/setup-helm@v1
with:
version: '3.11.1'
- name: Install Helm Unit Test Plugin
run: |
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
- name: Run unit tests
run:
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm

48
.github/workflows/interpret_load_test.py vendored
View file

@ -52,7 +52,43 @@ def interpret_results(csv_file):
return markdown_table
def _get_docker_run_command_stable_release(release_version):
return f"""
\n\n
## Docker Run LiteLLM Proxy
```
docker run \\
-e STORE_MODEL_IN_DB=True \\
-p 4000:4000 \\
ghcr.io/berriai/litellm:litellm_stable_release_branch-{release_version}
```
"""
def _get_docker_run_command(release_version):
return f"""
\n\n
## Docker Run LiteLLM Proxy
```
docker run \\
-e STORE_MODEL_IN_DB=True \\
-p 4000:4000 \\
ghcr.io/berriai/litellm:main-{release_version}
```
"""
def get_docker_run_command(release_version):
if "stable" in release_version:
return _get_docker_run_command_stable_release(release_version)
else:
return _get_docker_run_command(release_version)
if __name__ == "__main__":
return
csv_file = "load_test_stats.csv" # Change this to the path of your CSV file
markdown_table = interpret_results(csv_file)
@ -64,6 +100,11 @@ if __name__ == "__main__":
) # Replace with your repository's username and name
latest_release = repo.get_latest_release()
print("got latest release: ", latest_release)
print(latest_release.title)
print(latest_release.tag_name)
release_version = latest_release.title
print("latest release body: ", latest_release.body)
print("markdown table: ", markdown_table)
@ -74,8 +115,15 @@ if __name__ == "__main__":
start_index = latest_release.body.find("Load Test LiteLLM Proxy Results")
existing_release_body = latest_release.body[:start_index]
docker_run_command = get_docker_run_command(release_version)
print("docker run command: ", docker_run_command)
new_release_body = (
existing_release_body
+ docker_run_command
+ "\n\n"
+ "### Don't want to maintain your internal proxy? get in touch 🎉"
+ "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
+ "\n\n"
+ "## Load Test LiteLLM Proxy Results"
+ "\n\n"

64
.github/workflows/issue-keyword-labeler.yml vendored Normal file
View file

@ -0,0 +1,64 @@
name: Issue Keyword Labeler
on:
issues:
types:
- opened
jobs:
scan-and-label:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Scan for provider keywords
id: scan
env:
PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic,gemini,cohere,mistral,groq,ollama,deepseek
run: python3 .github/scripts/scan_keywords.py
- name: Ensure label exists
if: steps.scan.outputs.found == 'true'
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'llm translation';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'c1ff72',
description: 'Issues related to LLM provider translation/mapping'
});
} else {
throw error;
}
}
- name: Add label to the issue
if: steps.scan.outputs.found == 'true'
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: ['llm translation']
});

116
.github/workflows/label-component.yml vendored Normal file
View file

@ -0,0 +1,116 @@
name: Label Component Issues
on:
issues:
types:
- opened
jobs:
add-component-label:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Add component labels
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const body = context.payload.issue.body;
if (!body) return;
// Define component mappings with regex patterns that handle flexible whitespace
const components = [
{
pattern: /What part of LiteLLM is this about\?\s*SDK \(litellm Python package\)/,
label: 'sdk',
color: '0E7C86',
description: 'Issues related to the litellm Python SDK'
},
{
pattern: /What part of LiteLLM is this about\?\s*Proxy/,
label: 'proxy',
color: '5319E7',
description: 'Issues related to the LiteLLM Proxy'
},
{
pattern: /What part of LiteLLM is this about\?\s*UI Dashboard/,
label: 'ui-dashboard',
color: 'D876E3',
description: 'Issues related to the LiteLLM UI Dashboard'
},
{
pattern: /What part of LiteLLM is this about\?\s*Docs/,
label: 'docs',
color: 'FBCA04',
description: 'Issues related to LiteLLM documentation'
}
];
// Find matching component
for (const component of components) {
if (component.pattern.test(body)) {
// Ensure label exists
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: component.label
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: component.label,
color: component.color,
description: component.description
});
}
}
// Add label to issue
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [component.label]
});
break;
}
}
// Check for 'claude code' keyword (can be applied alongside component labels)
if (/claude code/i.test(body)) {
const claudeLabel = {
name: 'claude code',
color: '7c3aed',
description: 'Issues related to Claude Code usage'
};
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: claudeLabel.name
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: claudeLabel.name,
color: claudeLabel.color,
description: claudeLabel.description
});
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [claudeLabel.name]
});
}

35
.github/workflows/lint.yml vendored
View file

@ -1,35 +0,0 @@
name: Lint
# If a pull-request is pushed then cancel all previously running jobs related
# to that pull-request
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
on:
push:
branches:
- main
pull_request:
branches:
- main
- master
env:
PY_COLORS: 1
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
- name: Install poetry
run: pipx install poetry
- uses: actions/setup-python@v5.0.0
with:
python-version: '3.12'
- run: poetry install
- name: Run flake8 (https://flake8.pycqa.org/en/latest/)
run: poetry run flake8

89
.github/workflows/llm-translation-testing.yml vendored Normal file
View file

@ -0,0 +1,89 @@
name: LLM Translation Tests
on:
workflow_dispatch:
inputs:
release_candidate_tag:
description: 'Release candidate tag/version'
required: true
type: string
push:
tags:
- 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1
jobs:
run-llm-translation-tests:
runs-on: ubuntu-latest
timeout-minutes: 90
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: latest
virtualenvs-create: true
virtualenvs-in-project: true
- name: Cache Poetry dependencies
uses: actions/cache@v3
with:
path: |
~/.cache/pypoetry
.venv
key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
restore-keys: |
${{ runner.os }}-poetry-
- name: Install dependencies
run: |
poetry install --with dev
poetry run pip install pytest-xdist pytest-timeout
- name: Create test results directory
run: mkdir -p test-results
- name: Run LLM Translation Tests
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
# Add other API keys as needed
run: |
python .github/workflows/run_llm_translation_tests.py \
--tag "${{ github.event.inputs.release_candidate_tag || github.ref_name }}" \
--commit "${{ github.sha }}" \
|| true # Continue even if tests fail
- name: Display test summary
if: always()
run: |
if [ -f "test-results/llm_translation_report.md" ]; then
echo "Test report generated successfully!"
echo "Artifact will contain:"
echo "- test-results/junit.xml (JUnit XML results)"
echo "- test-results/llm_translation_report.md (Beautiful markdown report)"
else
echo "Warning: Test report was not generated"
fi
- name: Upload test artifacts
uses: actions/upload-artifact@v4
if: always()
with:
name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
path: test-results/
retention-days: 30

15
.github/workflows/load_test.yml vendored
View file

@ -22,14 +22,23 @@ jobs:
run: |
python -m pip install --upgrade pip
pip install PyGithub
- name: re-deploy proxy
run: |
echo "Current working directory: $PWD"
ls
python ".github/workflows/redeploy_proxy.py"
env:
LOAD_TEST_REDEPLOY_URL1: ${{ secrets.LOAD_TEST_REDEPLOY_URL1 }}
LOAD_TEST_REDEPLOY_URL2: ${{ secrets.LOAD_TEST_REDEPLOY_URL2 }}
working-directory: ${{ github.workspace }}
- name: Run Load Test
id: locust_run
uses: BerriAI/locust-github-action@master
with:
LOCUSTFILE: ".github/workflows/locustfile.py"
URL: "https://litellm-database-docker-build-production.up.railway.app/"
USERS: "100"
RATE: "10"
URL: "https://post-release-load-test-proxy.onrender.com/"
USERS: "20"
RATE: "20"
RUNTIME: "300s"
- name: Process Load Test Stats
run: |

18
.github/workflows/locustfile.py vendored
View file

@ -1,6 +1,4 @@
from locust import HttpUser, task, between, events
import json
import time
from locust import HttpUser, task, between
class MyUser(HttpUser):
@ -10,7 +8,7 @@ class MyUser(HttpUser):
def chat_completion(self):
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer sk-S2-EZTUUDY0EmM6-Fy0Fyw",
"Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg",
# Include any additional headers you may need for authentication, etc.
}
@ -28,15 +26,3 @@ class MyUser(HttpUser):
response = self.client.post("chat/completions", json=payload, headers=headers)
# Print or log the response if needed
@task(10)
def health_readiness(self):
start_time = time.time()
response = self.client.get("health/readiness")
response_time = time.time() - start_time
@task(10)
def health_liveliness(self):
start_time = time.time()
response = self.client.get("health/liveliness")
response_time = time.time() - start_time

34
.github/workflows/main.yml vendored Normal file
View file

@ -0,0 +1,34 @@
name: Publish Dev Release to PyPI
on:
workflow_dispatch:
jobs:
publish-dev-release:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.8 # Adjust the Python version as needed
- name: Install dependencies
run: pip install toml twine
- name: Read version from pyproject.toml
id: read-version
run: |
version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])')
printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
- name: Check if version exists on PyPI
id: check-version
run: |
set -e
if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then
echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish."

207
.github/workflows/publish-migrations.yml vendored Normal file
View file

@ -0,0 +1,207 @@
name: Publish Prisma Migrations
permissions:
contents: write
pull-requests: write
on:
push:
paths:
- 'schema.prisma' # Check root schema.prisma
branches:
- main
jobs:
publish-migrations:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
services:
postgres:
image: postgres:14
env:
POSTGRES_DB: temp_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
# Add shadow database service
postgres_shadow:
image: postgres:14
env:
POSTGRES_DB: shadow_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
ports:
- 5433:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Install Dependencies
run: |
pip install prisma
pip install python-dotenv
- name: Generate Initial Migration if None Exists
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
run: |
mkdir -p deploy/migrations
echo 'provider = "postgresql"' > deploy/migrations/migration_lock.toml
if [ -z "$(ls -A deploy/migrations/2* 2>/dev/null)" ]; then
echo "No existing migrations found, creating baseline..."
VERSION=$(date +%Y%m%d%H%M%S)
mkdir -p deploy/migrations/${VERSION}_initial
echo "Generating initial migration..."
# Save raw output for debugging
prisma migrate diff \
--from-empty \
--to-schema-datamodel schema.prisma \
--shadow-database-url "${SHADOW_DATABASE_URL}" \
--script > deploy/migrations/${VERSION}_initial/raw_migration.sql
echo "Raw migration file content:"
cat deploy/migrations/${VERSION}_initial/raw_migration.sql
echo "Cleaning migration file..."
# Clean the file
sed '/^Installing/d' deploy/migrations/${VERSION}_initial/raw_migration.sql > deploy/migrations/${VERSION}_initial/migration.sql
# Verify the migration file
if [ ! -s deploy/migrations/${VERSION}_initial/migration.sql ]; then
echo "ERROR: Migration file is empty after cleaning"
echo "Original content was:"
cat deploy/migrations/${VERSION}_initial/raw_migration.sql
exit 1
fi
echo "Final migration file content:"
cat deploy/migrations/${VERSION}_initial/migration.sql
# Verify it starts with SQL
if ! head -n 1 deploy/migrations/${VERSION}_initial/migration.sql | grep -q "^--\|^CREATE\|^ALTER"; then
echo "ERROR: Migration file does not start with SQL command or comment"
echo "First line is:"
head -n 1 deploy/migrations/${VERSION}_initial/migration.sql
echo "Full content is:"
cat deploy/migrations/${VERSION}_initial/migration.sql
exit 1
fi
echo "Initial migration generated at $(date -u)" > deploy/migrations/${VERSION}_initial/README.md
fi
- name: Compare and Generate Migration
if: success()
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
run: |
# Create temporary migration workspace
mkdir -p temp_migrations
# Copy existing migrations (will not fail if directory is empty)
cp -r deploy/migrations/* temp_migrations/ 2>/dev/null || true
VERSION=$(date +%Y%m%d%H%M%S)
# Generate diff against existing migrations or empty state
prisma migrate diff \
--from-migrations temp_migrations \
--to-schema-datamodel schema.prisma \
--shadow-database-url "${SHADOW_DATABASE_URL}" \
--script > temp_migrations/migration_${VERSION}.sql
# Check if there are actual changes
if [ -s temp_migrations/migration_${VERSION}.sql ]; then
echo "Changes detected, creating new migration"
mkdir -p deploy/migrations/${VERSION}_schema_update
mv temp_migrations/migration_${VERSION}.sql deploy/migrations/${VERSION}_schema_update/migration.sql
echo "Migration generated at $(date -u)" > deploy/migrations/${VERSION}_schema_update/README.md
else
echo "No schema changes detected"
exit 0
fi
- name: Verify Migration
if: success()
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
run: |
# Create test database
psql "${SHADOW_DATABASE_URL}" -c 'CREATE DATABASE migration_test;'
# Apply all migrations in order to verify
for migration in deploy/migrations/*/migration.sql; do
echo "Applying migration: $migration"
psql "${SHADOW_DATABASE_URL}" -f $migration
done
# Add this step before create-pull-request to debug permissions
- name: Check Token Permissions
run: |
echo "Checking token permissions..."
curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/BerriAI/litellm/collaborators
echo "\nChecking if token can create PRs..."
curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/BerriAI/litellm
# Add this debug step before git push
- name: Debug Changed Files
run: |
echo "Files staged for commit:"
git diff --name-status --staged
echo "\nAll changed files:"
git status
- name: Create Pull Request
if: success()
uses: peter-evans/create-pull-request@v5
with:
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: "chore: update prisma migrations"
title: "Update Prisma Migrations"
body: |
Auto-generated migration based on schema.prisma changes.
Generated files:
- deploy/migrations/${VERSION}_schema_update/migration.sql
- deploy/migrations/${VERSION}_schema_update/README.md
branch: feat/prisma-migration-${{ env.VERSION }}
base: main
delete-branch: true
- name: Generate and Save Migrations
run: |
# Only add migration files
git add deploy/migrations/
git status # Debug what's being committed
git commit -m "chore: update prisma migrations"

20
.github/workflows/redeploy_proxy.py vendored Normal file
View file

@ -0,0 +1,20 @@
"""
redeploy_proxy.py
"""
import os
import requests
import time
# send a get request to this endpoint
deploy_hook1 = os.getenv("LOAD_TEST_REDEPLOY_URL1")
response = requests.get(deploy_hook1, timeout=20)
deploy_hook2 = os.getenv("LOAD_TEST_REDEPLOY_URL2")
response = requests.get(deploy_hook2, timeout=20)
print("SENT GET REQUESTS to re-deploy proxy")
print("sleeeping.... for 60s")
time.sleep(60)

39
.github/workflows/reset_stable.yml vendored Normal file
View file

@ -0,0 +1,39 @@
name: Reset litellm_stable branch
on:
release:
types: [published, created]
jobs:
update-stable-branch:
if: ${{ startsWith(github.event.release.tag_name, 'v') && !endsWith(github.event.release.tag_name, '-stable') }}
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Reset litellm_stable_release_branch branch to the release commit
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Configure Git user
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
# Fetch all branches and tags
git fetch --all
# Check if the litellm_stable_release_branch branch exists
if git show-ref --verify --quiet refs/remotes/origin/litellm_stable_release_branch; then
echo "litellm_stable_release_branch branch exists."
git checkout litellm_stable_release_branch
else
echo "litellm_stable_release_branch branch does not exist. Creating it."
git checkout -b litellm_stable_release_branch
fi
# Reset litellm_stable_release_branch branch to the release commit
git reset --hard $GITHUB_SHA
# Push the updated litellm_stable_release_branch branch
git push origin litellm_stable_release_branch --force

439
.github/workflows/run_llm_translation_tests.py vendored Executable file
View file

@ -0,0 +1,439 @@
#!/usr/bin/env python3
"""
Run LLM Translation Tests and Generate Beautiful Markdown Report
This script runs the LLM translation tests and generates a comprehensive
markdown report with provider-specific breakdowns and test statistics.
"""
import os
import sys
import subprocess
import xml.etree.ElementTree as ET
from collections import defaultdict
from datetime import datetime
from pathlib import Path
import json
from typing import Dict, List, Tuple, Optional
# ANSI color codes for terminal output
class Colors:
GREEN = '\033[92m'
RED = '\033[91m'
YELLOW = '\033[93m'
BLUE = '\033[94m'
PURPLE = '\033[95m'
CYAN = '\033[96m'
RESET = '\033[0m'
BOLD = '\033[1m'
def print_colored(message: str, color: str = Colors.RESET):
"""Print colored message to terminal"""
print(f"{color}{message}{Colors.RESET}")
def get_provider_from_test_file(test_file: str) -> str:
"""Map test file names to provider names"""
provider_mapping = {
'test_anthropic': 'Anthropic',
'test_azure': 'Azure',
'test_bedrock': 'AWS Bedrock',
'test_openai': 'OpenAI',
'test_vertex': 'Google Vertex AI',
'test_gemini': 'Google Vertex AI',
'test_cohere': 'Cohere',
'test_databricks': 'Databricks',
'test_groq': 'Groq',
'test_together': 'Together AI',
'test_mistral': 'Mistral',
'test_deepseek': 'DeepSeek',
'test_replicate': 'Replicate',
'test_huggingface': 'HuggingFace',
'test_fireworks': 'Fireworks AI',
'test_perplexity': 'Perplexity',
'test_cloudflare': 'Cloudflare',
'test_voyage': 'Voyage AI',
'test_xai': 'xAI',
'test_nvidia': 'NVIDIA',
'test_watsonx': 'IBM watsonx',
'test_azure_ai': 'Azure AI',
'test_snowflake': 'Snowflake',
'test_infinity': 'Infinity',
'test_jina': 'Jina AI',
'test_deepgram': 'Deepgram',
'test_clarifai': 'Clarifai',
'test_triton': 'Triton',
}
for key, provider in provider_mapping.items():
if key in test_file:
return provider
# For cross-provider test files
if any(name in test_file for name in ['test_optional_params', 'test_prompt_factory',
'test_router', 'test_text_completion']):
return f'Cross-Provider Tests ({test_file})'
return 'Other Tests'
def format_duration(seconds: float) -> str:
"""Format duration in human-readable format"""
if seconds < 60:
return f"{seconds:.2f}s"
elif seconds < 3600:
minutes = int(seconds // 60)
secs = seconds % 60
return f"{minutes}m {secs:.0f}s"
else:
hours = int(seconds // 3600)
minutes = int((seconds % 3600) // 60)
return f"{hours}h {minutes}m"
def generate_markdown_report(junit_xml_path: str, output_path: str, tag: str = None, commit: str = None):
"""Generate a beautiful markdown report from JUnit XML"""
try:
tree = ET.parse(junit_xml_path)
root = tree.getroot()
# Handle both testsuite and testsuites root
if root.tag == 'testsuites':
suites = root.findall('testsuite')
else:
suites = [root]
# Overall statistics
total_tests = 0
total_failures = 0
total_errors = 0
total_skipped = 0
total_time = 0.0
# Provider breakdown
provider_stats = defaultdict(lambda: {'passed': 0, 'failed': 0, 'skipped': 0, 'errors': 0, 'time': 0.0})
provider_tests = defaultdict(list)
for suite in suites:
total_tests += int(suite.get('tests', 0))
total_failures += int(suite.get('failures', 0))
total_errors += int(suite.get('errors', 0))
total_skipped += int(suite.get('skipped', 0))
total_time += float(suite.get('time', 0))
for testcase in suite.findall('testcase'):
classname = testcase.get('classname', '')
test_name = testcase.get('name', '')
test_time = float(testcase.get('time', 0))
# Extract test file name from classname
if '.' in classname:
parts = classname.split('.')
test_file = parts[-2] if len(parts) > 1 else 'unknown'
else:
test_file = 'unknown'
provider = get_provider_from_test_file(test_file)
provider_stats[provider]['time'] += test_time
# Check test status
if testcase.find('failure') is not None:
provider_stats[provider]['failed'] += 1
failure = testcase.find('failure')
failure_msg = failure.get('message', '') if failure is not None else ''
provider_tests[provider].append({
'name': test_name,
'status': 'FAILED',
'time': test_time,
'message': failure_msg
})
elif testcase.find('error') is not None:
provider_stats[provider]['errors'] += 1
error = testcase.find('error')
error_msg = error.get('message', '') if error is not None else ''
provider_tests[provider].append({
'name': test_name,
'status': 'ERROR',
'time': test_time,
'message': error_msg
})
elif testcase.find('skipped') is not None:
provider_stats[provider]['skipped'] += 1
skip = testcase.find('skipped')
skip_msg = skip.get('message', '') if skip is not None else ''
provider_tests[provider].append({
'name': test_name,
'status': 'SKIPPED',
'time': test_time,
'message': skip_msg
})
else:
provider_stats[provider]['passed'] += 1
provider_tests[provider].append({
'name': test_name,
'status': 'PASSED',
'time': test_time,
'message': ''
})
passed = total_tests - total_failures - total_errors - total_skipped
# Generate the markdown report
with open(output_path, 'w') as f:
# Header
f.write("# LLM Translation Test Results\n\n")
# Metadata table
f.write("## Test Run Information\n\n")
f.write("| Field | Value |\n")
f.write("|-------|-------|\n")
f.write(f"| **Tag** | `{tag or 'N/A'}` |\n")
f.write(f"| **Date** | {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')} |\n")
f.write(f"| **Commit** | `{commit or 'N/A'}` |\n")
f.write(f"| **Duration** | {format_duration(total_time)} |\n")
f.write("\n")
# Overall statistics with visual elements
f.write("## Overall Statistics\n\n")
# Summary box
f.write("```\n")
f.write(f"Total Tests: {total_tests}\n")
f.write(f"├── Passed: {passed:>4} ({(passed/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
f.write(f"├── Failed: {total_failures:>4} ({(total_failures/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
f.write(f"├── Errors: {total_errors:>4} ({(total_errors/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
f.write(f"└── Skipped: {total_skipped:>4} ({(total_skipped/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
f.write("```\n\n")
# Provider summary table
f.write("## Results by Provider\n\n")
f.write("| Provider | Total | Pass | Fail | Error | Skip | Pass Rate | Duration |\n")
f.write("|----------|-------|------|------|-------|------|-----------|----------|")
# Sort providers: specific providers first, then cross-provider tests
sorted_providers = []
cross_provider = []
for p in sorted(provider_stats.keys()):
if 'Cross-Provider' in p or p == 'Other Tests':
cross_provider.append(p)
else:
sorted_providers.append(p)
all_providers = sorted_providers + cross_provider
for provider in all_providers:
stats = provider_stats[provider]
total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
pass_rate = (stats['passed'] / total * 100) if total > 0 else 0
f.write(f"\n| {provider} | {total} | {stats['passed']} | {stats['failed']} | ")
f.write(f"{stats['errors']} | {stats['skipped']} | {pass_rate:.1f}% | ")
f.write(f"{format_duration(stats['time'])} |")
# Detailed test results by provider
f.write("\n\n## Detailed Test Results\n\n")
for provider in sorted_providers:
if provider_tests[provider]:
stats = provider_stats[provider]
total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
f.write(f"### {provider}\n\n")
f.write(f"**Summary:** {stats['passed']}/{total} passed ")
f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%) ")
f.write(f"in {format_duration(stats['time'])}\n\n")
# Group tests by status
tests_by_status = defaultdict(list)
for test in provider_tests[provider]:
tests_by_status[test['status']].append(test)
# Show failed tests first (if any)
if tests_by_status['FAILED']:
f.write("<details>\n<summary>Failed Tests</summary>\n\n")
for test in tests_by_status['FAILED']:
f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
if test['message']:
# Truncate long error messages
msg = test['message'][:200] + '...' if len(test['message']) > 200 else test['message']
f.write(f" > {msg}\n")
f.write("\n</details>\n\n")
# Show errors (if any)
if tests_by_status['ERROR']:
f.write("<details>\n<summary>Error Tests</summary>\n\n")
for test in tests_by_status['ERROR']:
f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
f.write("\n</details>\n\n")
# Show passed tests in collapsible section
if tests_by_status['PASSED']:
f.write("<details>\n<summary>Passed Tests</summary>\n\n")
for test in tests_by_status['PASSED']:
f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
f.write("\n</details>\n\n")
# Show skipped tests (if any)
if tests_by_status['SKIPPED']:
f.write("<details>\n<summary>Skipped Tests</summary>\n\n")
for test in tests_by_status['SKIPPED']:
f.write(f"- `{test['name']}`\n")
f.write("\n</details>\n\n")
# Cross-provider tests in a separate section
if cross_provider:
f.write("### Cross-Provider Tests\n\n")
for provider in cross_provider:
if provider_tests[provider]:
stats = provider_stats[provider]
total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
f.write(f"#### {provider}\n\n")
f.write(f"**Summary:** {stats['passed']}/{total} passed ")
f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%)\n\n")
# For cross-provider tests, just show counts
f.write(f"- Passed: {stats['passed']}\n")
if stats['failed'] > 0:
f.write(f"- Failed: {stats['failed']}\n")
if stats['errors'] > 0:
f.write(f"- Errors: {stats['errors']}\n")
if stats['skipped'] > 0:
f.write(f"- Skipped: {stats['skipped']}\n")
f.write("\n")
print_colored(f"Report generated: {output_path}", Colors.GREEN)
except Exception as e:
print_colored(f"Error generating report: {e}", Colors.RED)
raise
def run_tests(test_path: str = "tests/llm_translation/",
junit_xml: str = "test-results/junit.xml",
report_path: str = "test-results/llm_translation_report.md",
tag: str = None,
commit: str = None) -> int:
"""Run the LLM translation tests and generate report"""
# Create test results directory
os.makedirs(os.path.dirname(junit_xml), exist_ok=True)
print_colored("Starting LLM Translation Tests", Colors.BOLD + Colors.BLUE)
print_colored(f"Test directory: {test_path}", Colors.CYAN)
print_colored(f"Output: {junit_xml}", Colors.CYAN)
print()
# Run pytest
cmd = [
"poetry", "run", "pytest", test_path,
f"--junitxml={junit_xml}",
"-v",
"--tb=short",
"--maxfail=500",
"-n", "auto"
]
# Add timeout if pytest-timeout is installed
try:
subprocess.run(["poetry", "run", "python", "-c", "import pytest_timeout"],
capture_output=True, check=True)
cmd.extend(["--timeout=300"])
except:
print_colored("Warning: pytest-timeout not installed, skipping timeout option", Colors.YELLOW)
print_colored("Running pytest with command:", Colors.YELLOW)
print(f" {' '.join(cmd)}")
print()
# Run the tests
result = subprocess.run(cmd, capture_output=False)
# Generate the report regardless of test outcome
if os.path.exists(junit_xml):
print()
print_colored("Generating test report...", Colors.BLUE)
generate_markdown_report(junit_xml, report_path, tag, commit)
# Print summary to console
print()
print_colored("Test Summary:", Colors.BOLD + Colors.PURPLE)
# Parse XML for quick summary
tree = ET.parse(junit_xml)
root = tree.getroot()
if root.tag == 'testsuites':
suites = root.findall('testsuite')
else:
suites = [root]
total = sum(int(s.get('tests', 0)) for s in suites)
failures = sum(int(s.get('failures', 0)) for s in suites)
errors = sum(int(s.get('errors', 0)) for s in suites)
skipped = sum(int(s.get('skipped', 0)) for s in suites)
passed = total - failures - errors - skipped
print(f" Total: {total}")
print_colored(f" Passed: {passed}", Colors.GREEN)
if failures > 0:
print_colored(f" Failed: {failures}", Colors.RED)
if errors > 0:
print_colored(f" Errors: {errors}", Colors.RED)
if skipped > 0:
print_colored(f" Skipped: {skipped}", Colors.YELLOW)
if total > 0:
pass_rate = (passed / total) * 100
color = Colors.GREEN if pass_rate >= 80 else Colors.YELLOW if pass_rate >= 60 else Colors.RED
print_colored(f" Pass Rate: {pass_rate:.1f}%", color)
else:
print_colored("No test results found!", Colors.RED)
print()
print_colored("Test run complete!", Colors.BOLD + Colors.GREEN)
return result.returncode
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Run LLM Translation Tests")
parser.add_argument("--test-path", default="tests/llm_translation/",
help="Path to test directory")
parser.add_argument("--junit-xml", default="test-results/junit.xml",
help="Path for JUnit XML output")
parser.add_argument("--report", default="test-results/llm_translation_report.md",
help="Path for markdown report")
parser.add_argument("--tag", help="Git tag or version")
parser.add_argument("--commit", help="Git commit SHA")
args = parser.parse_args()
# Get git info if not provided
if not args.commit:
try:
result = subprocess.run(["git", "rev-parse", "HEAD"],
capture_output=True, text=True)
if result.returncode == 0:
args.commit = result.stdout.strip()
except:
pass
if not args.tag:
try:
result = subprocess.run(["git", "describe", "--tags", "--abbrev=0"],
capture_output=True, text=True)
if result.returncode == 0:
args.tag = result.stdout.strip()
except:
pass
exit_code = run_tests(
test_path=args.test_path,
junit_xml=args.junit_xml,
report_path=args.report,
tag=args.tag,
commit=args.commit
)
sys.exit(exit_code)

67
.github/workflows/simple_pypi_publish.yml vendored Normal file
View file

@ -0,0 +1,67 @@
name: Simple PyPI Publish
on:
workflow_dispatch:
inputs:
version:
description: 'Version to publish (e.g., 1.74.10)'
required: true
type: string
env:
TWINE_USERNAME: __token__
jobs:
publish:
runs-on: ubuntu-latest
if: github.repository == 'BerriAI/litellm'
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install toml build wheel twine
- name: Update version in pyproject.toml
run: |
python -c "
import toml
with open('pyproject.toml', 'r') as f:
data = toml.load(f)
data['tool']['poetry']['version'] = '${{ github.event.inputs.version }}'
with open('pyproject.toml', 'w') as f:
toml.dump(data, f)
print(f'Updated version to ${{ github.event.inputs.version }}')
"
- name: Copy model prices file
run: |
cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
- name: Build package
run: |
rm -rf build dist
python -m build
- name: Publish to PyPI
env:
TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
run: |
twine upload dist/*
- name: Output success
run: |
echo "✅ Successfully published litellm v${{ github.event.inputs.version }} to PyPI"
echo "📦 Package: https://pypi.org/project/litellm/${{ github.event.inputs.version }}/"

20
.github/workflows/stale.yml vendored Normal file
View file

@ -0,0 +1,20 @@
name: "Stale Issue Management"
on:
schedule:
- cron: '0 0 * * *' # Runs daily at midnight UTC
workflow_dispatch:
jobs:
stale:
runs-on: ubuntu-latest
steps:
- uses: actions/stale@v8
with:
repo-token: "${{ secrets.GITHUB_TOKEN }}"
stale-issue-message: "This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
days-before-stale: 90 # Revert to 60 days
days-before-close: 7 # Revert to 7 days
stale-issue-label: "stale"
operations-per-run: 1000

76
.github/workflows/test-linting.yml vendored Normal file
View file

@ -0,0 +1,76 @@
name: LiteLLM Linting
on:
pull_request:
branches: [ main ]
jobs:
lint:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
clean: true
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Clean Python cache
run: |
find . -type d -name "__pycache__" -exec rm -rf {} + || true
find . -name "*.pyc" -delete || true
- name: Install dependencies
run: |
poetry lock
poetry install --with dev
poetry run pip install openai==1.100.1
- name: Run Black formatting
run: |
cd litellm
poetry run black .
cd ..
- name: Debug - Check file state
run: |
echo "Current branch:"
git branch --show-current
echo "Last 3 commits:"
git log --oneline -3
echo "File content around line 43:"
head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
- name: Run Ruff linting
run: |
cd litellm
poetry run ruff check .
cd ..
- name: Print OpenAI version
run: |
poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Run MyPy type checking
run: |
cd litellm
poetry run mypy .
cd ..
- name: Check for circular imports
run: |
cd litellm
poetry run python ../tests/documentation_tests/test_circular_imports.py
cd ..
- name: Check import safety
run: |
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)

46
.github/workflows/test-litellm.yml vendored Normal file
View file

@ -0,0 +1,46 @@
name: LiteLLM Mock Tests (folder - tests/test_litellm)
on:
pull_request:
branches: [ main ]
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- name: Thank You Message
run: |
echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Install dependencies
run: |
poetry lock
poetry install --with dev,proxy-dev --extras "proxy semantic-router"
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
poetry run pip install "google-genai==1.22.0"
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.22"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
poetry run pip install -e .
cd ..
- name: Run tests
run: |
poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

49
.github/workflows/test-mcp.yml vendored Normal file
View file

@ -0,0 +1,49 @@
name: LiteLLM MCP Tests (folder - tests/mcp_tests)
on:
pull_request:
branches: [ main ]
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- name: Thank You Message
run: |
echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Install dependencies
run: |
poetry lock
poetry install --with dev,proxy-dev --extras "proxy semantic-router"
poetry run pip install "pytest==7.3.1"
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install "pytest-cov==5.0.0"
poetry run pip install "pytest-asyncio==0.21.1"
poetry run pip install "respx==0.22.0"
poetry run pip install "pydantic==2.11.0"
poetry run pip install "mcp==1.25.0"
poetry run pip install pytest-xdist
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
python -m pip install -e .
cd ..
- name: Run MCP tests
run: |
poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5

15
.github/workflows/test-model-map.yaml vendored Normal file
View file

@ -0,0 +1,15 @@
name: Validate model_prices_and_context_window.json
on:
pull_request:
branches: [ main ]
jobs:
validate-model-prices-json:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Validate model_prices_and_context_window.json
run: |
jq empty model_prices_and_context_window.json

52
.gitignore vendored
View file

@ -1,5 +1,10 @@
.python-version
.venv
.venv_policy_test
.env
.newenv
newenv/*
litellm/proxy/myenv/*
litellm_uuid.txt
__pycache__/
*.pyc
@ -45,8 +50,53 @@ deploy/charts/litellm/charts/*
deploy/charts/*.tgz
litellm/proxy/vertex_key.json
**/.vim/
/node_modules
**/node_modules
kub.yaml
loadtest_kub.yaml
litellm/proxy/_new_secret_config.yaml
litellm/proxy/_new_secret_config.yaml
litellm/proxy/_super_secret_config.yaml
litellm/proxy/_super_secret_config.yaml
litellm/proxy/myenv/bin/activate
litellm/proxy/myenv/bin/Activate.ps1
myenv/*
litellm/tests/log.txt
litellm/tests/langfuse.log
litellm/tests/langfuse.log
litellm/proxy/google-cloud-sdk/*
tests/llm_translation/log.txt
venv/
tests/local_testing/log.txt
.codegpt
litellm/proxy/_new_new_secret_config.yaml
litellm/proxy/custom_guardrail.py
.mypy_cache/*
.mypy_cache/*
litellm/proxy/application.log
tests/llm_translation/vertex_test_account.json
tests/llm_translation/test_vertex_key.json
litellm/proxy/migrations/0_init/migration.sql
litellm/proxy/db/migrations/0_init/migration.sql
litellm/proxy/db/migrations/*
litellm/proxy/migrations/*config.yaml
litellm/proxy/migrations/*
litellm/proxy/to_delete_loadtest_work/*
config.yaml
tests/litellm/litellm_core_utils/llm_cost_calc/log.txt
tests/test_custom_dir/*
test.py
litellm_config.yaml
.cursor
.vscode/launch.json
litellm/proxy/to_delete_loadtest_work/*
update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
scripts/test_vertex_ai_search.py
LAZY_LOADING_IMPROVEMENTS.md
STABILIZATION_TODO.md
**/test-results
**/playwright-report
**/*.storageState.json
**/coverage

42
.pre-commit-config.yaml
View file

@ -1,26 +1,40 @@
repos:
- repo: https://github.com/psf/black
rev: 24.2.0
- repo: local
hooks:
- id: black
- id: pyright
name: pyright
entry: pyright
language: system
types: [python]
files: ^(litellm/|litellm_proxy_extras/|enterprise/)
- id: isort
name: isort
entry: isort
language: system
types: [python]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
exclude: ^litellm/__init__.py$
# - id: black
# name: black
# entry: poetry run black
# language: system
# types: [python]
# files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- repo: https://github.com/pycqa/flake8
rev: 7.0.0 # The version of flake8 to use
hooks:
- id: flake8
exclude: ^litellm/tests/|^litellm/proxy/proxy_cli.py|^litellm/integrations/|^litellm/proxy/tests/
exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/
additional_dependencies: [flake8-print]
files: litellm/.*\.py
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- repo: https://github.com/python-poetry/poetry
rev: 1.8.0
hooks:
- id: poetry-check
files: ^(pyproject.toml|litellm-proxy-extras/pyproject.toml)$
- repo: local
hooks:
- id: check-files-match
name: Check if files match
entry: python3 ci_cd/check_files_match.py
language: system
- repo: local
hooks:
- id: mypy
name: mypy
entry: python3 -m mypy --ignore-missing-imports
language: system
types: [python]
files: ^litellm/
language: system

12
.trivyignore Normal file
View file

@ -0,0 +1,12 @@
# LiteLLM Trivy Ignore File
# CVEs listed here are temporarily allowlisted pending fixes
# Next.js vulnerabilities in UI dashboard (next@14.2.35)
# Allowlisted: 2026-01-31, 7-day fix timeline
# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+
# HIGH: DoS via request deserialization
GHSA-h25m-26qc-wcjf
# MEDIUM: Image Optimizer DoS
CVE-2025-59471

190
AGENTS.md Normal file
View file

@ -0,0 +1,190 @@
# INSTRUCTIONS FOR LITELLM
This document provides comprehensive instructions for AI agents working in the LiteLLM repository.
## OVERVIEW
LiteLLM is a unified interface for 100+ LLMs that:
- Translates inputs to provider-specific completion, embedding, and image generation endpoints
- Provides consistent OpenAI-format output across all providers
- Includes retry/fallback logic across multiple deployments (Router)
- Offers a proxy server (LLM Gateway) with budgets, rate limits, and authentication
- Supports advanced features like function calling, streaming, caching, and observability
## REPOSITORY STRUCTURE
### Core Components
- `litellm/` - Main library code
- `llms/` - Provider-specific implementations (OpenAI, Anthropic, Azure, etc.)
- `proxy/` - Proxy server implementation (LLM Gateway)
- `router_utils/` - Load balancing and fallback logic
- `types/` - Type definitions and schemas
- `integrations/` - Third-party integrations (observability, caching, etc.)
### Key Directories
- `tests/` - Comprehensive test suites
- `docs/my-website/` - Documentation website
- `ui/litellm-dashboard/` - Admin dashboard UI
- `enterprise/` - Enterprise-specific features
## DEVELOPMENT GUIDELINES
### MAKING CODE CHANGES
1. **Provider Implementations**: When adding/modifying LLM providers:
- Follow existing patterns in `litellm/llms/{provider}/`
- Implement proper transformation classes that inherit from `BaseConfig`
- Support both sync and async operations
- Handle streaming responses appropriately
- Include proper error handling with provider-specific exceptions
2. **Type Safety**:
- Use proper type hints throughout
- Update type definitions in `litellm/types/`
- Ensure compatibility with both Pydantic v1 and v2
3. **Testing**:
- Add tests in appropriate `tests/` subdirectories
- Include both unit tests and integration tests
- Test provider-specific functionality thoroughly
- Consider adding load tests for performance-critical changes
### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
1. **Tremor is DEPRECATED, do not use Tremor components in new features/changes**
- The only exception is the Tremor Table component and its required Tremor Table sub components.
2. **Use Common Components as much as possible**:
- These are usually defined in the `common_components` directory
- Use these components as much as possible and avoid building new components unless needed
3. **Testing**:
- The codebase uses **Vitest** and **React Testing Library**
- **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
- **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)
- **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled
- **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present
- **Test names must start with "should"**: All test names should follow the pattern `it("should ...")`
- **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed
- **Structure tests properly**:
- First test should verify the component renders successfully
- Subsequent tests should focus on functionality and user interactions
- Use `waitFor` for async operations that aren't already awaited
- **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation
### IMPORTANT PATTERNS
1. **Function/Tool Calling**:
- LiteLLM standardizes tool calling across providers
- OpenAI format is the standard, with transformations for other providers
- See `litellm/llms/anthropic/chat/transformation.py` for complex tool handling
2. **Streaming**:
- All providers should support streaming where possible
- Use consistent chunk formatting across providers
- Handle both sync and async streaming
3. **Error Handling**:
- Use provider-specific exception classes
- Maintain consistent error formats across providers
- Include proper retry logic and fallback mechanisms
4. **Configuration**:
- Support both environment variables and programmatic configuration
- Use `BaseConfig` classes for provider configurations
- Allow dynamic parameter passing
## PROXY SERVER (LLM GATEWAY)
The proxy server is a critical component that provides:
- Authentication and authorization
- Rate limiting and budget management
- Load balancing across multiple models/deployments
- Observability and logging
- Admin dashboard UI
- Enterprise features
Key files:
- `litellm/proxy/proxy_server.py` - Main server implementation
- `litellm/proxy/auth/` - Authentication logic
- `litellm/proxy/management_endpoints/` - Admin API endpoints
## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
LiteLLM supports MCP for agent workflows:
- MCP server integration for tool calling
- Transformation between OpenAI and MCP tool formats
- Support for external MCP servers (Zapier, Jira, Linear, etc.)
- See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/`
## RUNNING SCRIPTS
Use `poetry run python script.py` to run Python scripts in the project environment (for non-test files).
## GITHUB TEMPLATES
When opening issues or pull requests, follow these templates:
### Bug Reports (`.github/ISSUE_TEMPLATE/bug_report.yml`)
- Describe what happened vs. expected behavior
- Include relevant log output
- Specify LiteLLM version
- Indicate if you're part of an ML Ops team (helps with prioritization)
### Feature Requests (`.github/ISSUE_TEMPLATE/feature_request.yml`)
- Clearly describe the feature
- Explain motivation and use case with concrete examples
### Pull Requests (`.github/pull_request_template.md`)
- Add at least 1 test in `tests/litellm/`
- Ensure `make test-unit` passes
## TESTING CONSIDERATIONS
1. **Provider Tests**: Test against real provider APIs when possible
2. **Proxy Tests**: Include authentication, rate limiting, and routing tests
3. **Performance Tests**: Load testing for high-throughput scenarios
4. **Integration Tests**: End-to-end workflows including tool calling
## DOCUMENTATION
- Keep documentation in sync with code changes
- Update provider documentation when adding new providers
- Include code examples for new features
- Update changelog and release notes
## SECURITY CONSIDERATIONS
- Handle API keys securely
- Validate all inputs, especially for proxy endpoints
- Consider rate limiting and abuse prevention
- Follow security best practices for authentication
## ENTERPRISE FEATURES
- Some features are enterprise-only
- Check `enterprise/` directory for enterprise-specific code
- Maintain compatibility between open-source and enterprise versions
## COMMON PITFALLS TO AVOID
1. **Breaking Changes**: LiteLLM has many users - avoid breaking existing APIs
2. **Provider Specifics**: Each provider has unique quirks - handle them properly
3. **Rate Limits**: Respect provider rate limits in tests
4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
5. **Dependencies**: Keep dependencies minimal and well-justified
## HELPFUL RESOURCES
- Main documentation: https://docs.litellm.ai/
- Provider-specific docs in `docs/my-website/docs/providers/`
- Admin UI for testing proxy features
## WHEN IN DOUBT
- Follow existing patterns in the codebase
- Check similar provider implementations
- Ensure comprehensive test coverage
- Update documentation appropriately
- Consider backward compatibility impact

398
ARCHITECTURE.md Normal file
View file

@ -0,0 +1,398 @@
# LiteLLM Architecture - LiteLLM SDK + AI Gateway
This document helps contributors understand where to make changes in LiteLLM.
---
## How It Works
The LiteLLM AI Gateway (Proxy) uses the LiteLLM SDK internally for all LLM calls:
```
OpenAI SDK (client) ──▶ LiteLLM AI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
Anthropic SDK (client) ──▶ LiteLLMAI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
Any HTTP client ──▶ LiteLLMAI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
```
The **AI Gateway** adds authentication, rate limiting, budgets, and routing on top of the SDK.
The **SDK** handles the actual LLM provider calls, request/response transformations, and streaming.
---
## 1. AI Gateway (Proxy) Request Flow
The AI Gateway (`litellm/proxy/`) wraps the SDK with authentication, rate limiting, and management features.
```mermaid
sequenceDiagram
participant Client
participant ProxyServer as proxy/proxy_server.py
participant Auth as proxy/auth/user_api_key_auth.py
participant Redis as Redis Cache
participant Hooks as proxy/hooks/
participant Router as router.py
participant Main as main.py + utils.py
participant Handler as llms/custom_httpx/llm_http_handler.py
participant Transform as llms/{provider}/chat/transformation.py
participant Provider as LLM Provider API
participant CostCalc as cost_calculator.py
participant LoggingObj as litellm_logging.py
participant DBWriter as db/db_spend_update_writer.py
participant Postgres as PostgreSQL
%% Request Flow
Client->>ProxyServer: POST /v1/chat/completions
ProxyServer->>Auth: user_api_key_auth()
Auth->>Redis: Check API key cache
Redis-->>Auth: Key info + spend limits
ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter
Hooks->>Redis: Check/increment rate limit counters
ProxyServer->>Router: route_request()
Router->>Main: litellm.acompletion()
Main->>Handler: BaseLLMHTTPHandler.completion()
Handler->>Transform: ProviderConfig.transform_request()
Handler->>Provider: HTTP Request
Provider-->>Handler: Response
Handler->>Transform: ProviderConfig.transform_response()
Transform-->>Handler: ModelResponse
Handler-->>Main: ModelResponse
%% Cost Attribution (in utils.py wrapper)
Main->>LoggingObj: update_response_metadata()
LoggingObj->>CostCalc: _response_cost_calculator()
CostCalc->>CostCalc: completion_cost(tokens × price)
CostCalc-->>LoggingObj: response_cost
LoggingObj-->>Main: Set response._hidden_params["response_cost"]
Main-->>ProxyServer: ModelResponse (with cost in _hidden_params)
%% Response Headers + Async Logging
ProxyServer->>ProxyServer: Extract cost from hidden_params
ProxyServer->>LoggingObj: async_success_handler()
LoggingObj->>Hooks: async_log_success_event()
Hooks->>DBWriter: update_database(response_cost)
DBWriter->>Redis: Queue spend increment
DBWriter->>Postgres: Batch write spend logs (async)
ProxyServer-->>Client: ModelResponse + x-litellm-response-cost header
```
### Proxy Components
```mermaid
graph TD
subgraph "Incoming Request"
Client["POST /v1/chat/completions"]
end
subgraph "proxy/proxy_server.py"
Endpoint["chat_completion()"]
end
subgraph "proxy/auth/"
Auth["user_api_key_auth()"]
end
subgraph "proxy/"
PreCall["litellm_pre_call_utils.py"]
RouteRequest["route_llm_request.py"]
end
subgraph "litellm/"
Router["router.py"]
Main["main.py"]
end
subgraph "Infrastructure"
DualCache["DualCache<br/>(in-memory + Redis)"]
Postgres["PostgreSQL<br/>(keys, teams, spend logs)"]
end
Client --> Endpoint
Endpoint --> Auth
Auth --> DualCache
DualCache -.->|cache miss| Postgres
Auth --> PreCall
PreCall --> RouteRequest
RouteRequest --> Router
Router --> DualCache
Router --> Main
Main --> Client
```
**Key proxy files:**
- `proxy/proxy_server.py` - Main API endpoints
- `proxy/auth/` - Authentication (API keys, JWT, OAuth2)
- `proxy/hooks/` - Proxy-level callbacks
- `router.py` - Load balancing, fallbacks
- `router_strategy/` - Routing algorithms (`lowest_latency.py`, `simple_shuffle.py`, etc.)
**LLM-specific proxy endpoints:**
| Endpoint | Directory | Purpose |
|----------|-----------|---------|
| `/v1/messages` | `proxy/anthropic_endpoints/` | Anthropic Messages API |
| `/vertex-ai/*` | `proxy/vertex_ai_endpoints/` | Vertex AI passthrough |
| `/gemini/*` | `proxy/google_endpoints/` | Google AI Studio passthrough |
| `/v1/images/*` | `proxy/image_endpoints/` | Image generation |
| `/v1/batches` | `proxy/batches_endpoints/` | Batch processing |
| `/v1/files` | `proxy/openai_files_endpoints/` | File uploads |
| `/v1/fine_tuning` | `proxy/fine_tuning_endpoints/` | Fine-tuning jobs |
| `/v1/rerank` | `proxy/rerank_endpoints/` | Reranking |
| `/v1/responses` | `proxy/response_api_endpoints/` | OpenAI Responses API |
| `/v1/vector_stores` | `proxy/vector_store_endpoints/` | Vector stores |
| `/*` (passthrough) | `proxy/pass_through_endpoints/` | Direct provider passthrough |
**Proxy Hooks** (`proxy/hooks/__init__.py`):
| Hook | File | Purpose |
|------|------|---------|
| `max_budget_limiter` | `proxy/hooks/max_budget_limiter.py` | Enforce budget limits |
| `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user |
| `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation |
| `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation |
| `litellm_skills` | `proxy/hooks/skills_injection.py` | Skills injection |
To add a new proxy hook, implement `CustomLogger` and register in `PROXY_HOOKS`.
### Infrastructure Components
The AI Gateway uses external infrastructure for persistence and caching:
```mermaid
graph LR
subgraph "AI Gateway (proxy/)"
Proxy["proxy_server.py"]
Auth["auth/user_api_key_auth.py"]
DBWriter["db/db_spend_update_writer.py<br/>DBSpendUpdateWriter"]
InternalCache["utils.py<br/>InternalUsageCache"]
CostCallback["hooks/proxy_track_cost_callback.py<br/>_ProxyDBLogger"]
Scheduler["APScheduler<br/>ProxyStartupEvent"]
end
subgraph "SDK (litellm/)"
Router["router.py<br/>Router.cache (DualCache)"]
LLMCache["caching/caching_handler.py<br/>LLMCachingHandler"]
CacheClass["caching/caching.py<br/>Cache"]
end
subgraph "Redis (caching/redis_cache.py)"
RateLimit["Rate Limit Counters"]
SpendQueue["Spend Increment Queue"]
KeyCache["API Key Cache"]
TPM_RPM["TPM/RPM Tracking"]
Cooldowns["Deployment Cooldowns"]
LLMResponseCache["LLM Response Cache"]
end
subgraph "PostgreSQL (proxy/schema.prisma)"
Keys["LiteLLM_VerificationToken"]
Teams["LiteLLM_TeamTable"]
SpendLogs["LiteLLM_SpendLogs"]
Users["LiteLLM_UserTable"]
end
Auth --> InternalCache
InternalCache --> KeyCache
InternalCache -.->|cache miss| Keys
InternalCache --> RateLimit
Router --> TPM_RPM
Router --> Cooldowns
LLMCache --> CacheClass
CacheClass --> LLMResponseCache
CostCallback --> DBWriter
DBWriter --> SpendQueue
DBWriter --> SpendLogs
Scheduler --> SpendLogs
Scheduler --> Keys
```
| Component | Purpose | Key Files/Classes |
|-----------|---------|-------------------|
| **Redis** | Rate limiting, API key caching, TPM/RPM tracking, cooldowns, LLM response caching, spend queuing | `caching/redis_cache.py` (`RedisCache`), `caching/dual_cache.py` (`DualCache`) |
| **PostgreSQL** | API keys, teams, users, spend logs | `proxy/utils.py` (`PrismaClient`), `proxy/schema.prisma` |
| **InternalUsageCache** | Proxy-level cache for rate limits + API keys (in-memory + Redis) | `proxy/utils.py` (`InternalUsageCache`) |
| **Router.cache** | TPM/RPM tracking, deployment cooldowns, client caching (in-memory + Redis) | `router.py` (`Router.cache: DualCache`) |
| **LLMCachingHandler** | SDK-level LLM response/embedding caching | `caching/caching_handler.py` (`LLMCachingHandler`), `caching/caching.py` (`Cache`) |
| **DBSpendUpdateWriter** | Batches spend updates to reduce DB writes | `proxy/db/db_spend_update_writer.py` (`DBSpendUpdateWriter`) |
| **Cost Tracking** | Calculates and logs response costs | `proxy/hooks/proxy_track_cost_callback.py` (`_ProxyDBLogger`) |
**Background Jobs** (APScheduler, initialized in `proxy/proxy_server.py` → `ProxyStartupEvent.initialize_scheduled_background_jobs()`):
| Job | Interval | Purpose | Key Files |
|-----|----------|---------|-----------|
| `update_spend` | 60s | Batch write spend logs to PostgreSQL | `proxy/db/db_spend_update_writer.py` |
| `reset_budget` | 10-12min | Reset budgets for keys/users/teams | `proxy/management_helpers/budget_reset_job.py` |
| `add_deployment` | 10s | Sync new model deployments from DB | `proxy/proxy_server.py` (`ProxyConfig`) |
| `cleanup_old_spend_logs` | cron/interval | Delete old spend logs | `proxy/management_helpers/spend_log_cleanup.py` |
| `check_batch_cost` | 30min | Calculate costs for batch jobs | `proxy/management_helpers/check_batch_cost_job.py` |
| `check_responses_cost` | 30min | Calculate costs for responses API | `proxy/management_helpers/check_responses_cost_job.py` |
| `process_rotations` | 1hr | Auto-rotate API keys | `proxy/management_helpers/key_rotation_manager.py` |
| `_run_background_health_check` | continuous | Health check model deployments | `proxy/proxy_server.py` |
| `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
| `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
**Cost Attribution Flow:**
1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes
2. `update_response_metadata()` (`llm_response_utils/response_metadata.py`) is called
3. `logging_obj._response_cost_calculator()` (`litellm_logging.py`) calculates cost via `litellm.completion_cost()` (`cost_calculator.py`)
4. Cost is stored in `response._hidden_params["response_cost"]`
5. `proxy/common_request_processing.py` extracts cost from `hidden_params` and adds to response headers (`x-litellm-response-cost`)
6. `logging_obj.async_success_handler()` triggers callbacks including `_ProxyDBLogger.async_log_success_event()`
7. `DBSpendUpdateWriter.update_database()` queues spend increments to Redis
8. Background job `update_spend` flushes queued spend to PostgreSQL every 60s
---
## 2. SDK Request Flow
The SDK (`litellm/`) provides the core LLM calling functionality used by both direct SDK users and the AI Gateway.
```mermaid
graph TD
subgraph "SDK Entry Points"
Completion["litellm.completion()"]
Messages["litellm.messages()"]
end
subgraph "main.py"
Main["completion()<br/>acompletion()"]
end
subgraph "utils.py"
GetProvider["get_llm_provider()"]
end
subgraph "llms/custom_httpx/"
Handler["llm_http_handler.py<br/>BaseLLMHTTPHandler"]
HTTP["http_handler.py<br/>HTTPHandler / AsyncHTTPHandler"]
end
subgraph "llms/{provider}/chat/"
TransformReq["transform_request()"]
TransformResp["transform_response()"]
end
subgraph "litellm_core_utils/"
Streaming["streaming_handler.py"]
end
subgraph "integrations/ (async, off main thread)"
Callbacks["custom_logger.py<br/>Langfuse, Datadog, etc."]
end
Completion --> Main
Messages --> Main
Main --> GetProvider
GetProvider --> Handler
Handler --> TransformReq
TransformReq --> HTTP
HTTP --> Provider["LLM Provider API"]
Provider --> HTTP
HTTP --> TransformResp
TransformResp --> Streaming
Streaming --> Response["ModelResponse"]
Response -.->|async| Callbacks
```
**Key SDK files:**
- `main.py` - Entry points: `completion()`, `acompletion()`, `embedding()`
- `utils.py` - `get_llm_provider()` resolves model → provider
- `llms/custom_httpx/llm_http_handler.py` - Central HTTP orchestrator
- `llms/custom_httpx/http_handler.py` - Low-level HTTP client
- `llms/{provider}/chat/transformation.py` - Provider-specific transformations
- `litellm_core_utils/streaming_handler.py` - Streaming response handling
- `integrations/` - Async callbacks (Langfuse, Datadog, etc.)
---
## 3. Translation Layer
When a request comes in, it goes through a **translation layer** that converts between API formats.
Each translation is isolated in its own file, making it easy to test and modify independently.
### Where to find translations
| Incoming API | Provider | Translation File |
|--------------|----------|------------------|
| `/v1/chat/completions` | Anthropic | `llms/anthropic/chat/transformation.py` |
| `/v1/chat/completions` | Bedrock Converse | `llms/bedrock/chat/converse_transformation.py` |
| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` |
| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |
### Example: Debugging prompt caching
If `/v1/messages` → Bedrock Converse prompt caching isn't working but Bedrock Invoke works:
1. **Bedrock Converse translation**: `llms/bedrock/chat/converse_transformation.py`
2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py`
3. Compare how each handles `cache_control` in `transform_request()`
### How translations work
Each provider has a `Config` class that inherits from `BaseConfig` (`llms/base_llm/chat/transformation.py`):
```python
class ProviderConfig(BaseConfig):
def transform_request(self, model, messages, optional_params, litellm_params, headers):
# Convert OpenAI format → Provider format
return {"messages": transformed_messages, ...}
def transform_response(self, model, raw_response, model_response, logging_obj, ...):
# Convert Provider format → OpenAI format
return ModelResponse(choices=[...], usage=Usage(...))
```
The `BaseLLMHTTPHandler` (`llms/custom_httpx/llm_http_handler.py`) calls these methods - you never need to modify the handler itself.
---
## 4. Adding/Modifying Providers
### To add a new provider:
1. Create `llms/{provider}/chat/transformation.py`
2. Implement `Config` class with `transform_request()` and `transform_response()`
3. Add tests in `tests/llm_translation/test_{provider}.py`
### To add a feature (e.g., prompt caching):
1. Find the translation file from the table above
2. Modify `transform_request()` to handle the new parameter
3. Add unit tests that verify the transformation
### Testing checklist
When adding a feature, verify it works across all paths:
| Test | File Pattern |
|------|--------------|
| OpenAI passthrough | `tests/llm_translation/test_openai*.py` |
| Anthropic direct | `tests/llm_translation/test_anthropic*.py` |
| Bedrock Invoke | `tests/llm_translation/test_bedrock*.py` |
| Bedrock Converse | `tests/llm_translation/test_bedrock*converse*.py` |
| Vertex AI | `tests/llm_translation/test_vertex*.py` |
| Gemini | `tests/llm_translation/test_gemini*.py` |
### Unit testing translations
Translations are designed to be unit testable without making API calls:
```python
from litellm.llms.bedrock.chat.converse_transformation import BedrockConverseConfig
def test_prompt_caching_transform():
config = BedrockConverseConfig()
result = config.transform_request(
model="anthropic.claude-3-opus",
messages=[{"role": "user", "content": "test", "cache_control": {"type": "ephemeral"}}],
optional_params={},
litellm_params={},
headers={}
)
assert "cachePoint" in str(result) # Verify cache_control was translated
```

109
CLAUDE.md Normal file
View file

@ -0,0 +1,109 @@
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Development Commands
### Installation
- `make install-dev` - Install core development dependencies
- `make install-proxy-dev` - Install proxy development dependencies with full feature set
- `make install-test-deps` - Install all test dependencies
### Testing
- `make test` - Run all tests
- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
- `make test-integration` - Run integration tests (excludes unit tests)
- `pytest tests/` - Direct pytest execution
### Code Quality
- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
- `make format` - Apply Black code formatting
- `make lint-ruff` - Run Ruff linting only
- `make lint-mypy` - Run MyPy type checking only
### Single Test Files
- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
### Running Scripts
- `poetry run python script.py` - Run Python scripts (use for non-test files)
### GitHub Issue & PR Templates
When contributing to the project, use the appropriate templates:
**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
- Describe what happened vs. what you expected
- Include relevant log output
- Specify your LiteLLM version
**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
- Describe the feature clearly
- Explain the motivation and use case
**Pull Requests** (`.github/pull_request_template.md`):
- Add at least 1 test in `tests/litellm/`
- Ensure `make test-unit` passes
## Architecture Overview
LiteLLM is a unified interface for 100+ LLM providers with two main components:
### Core Library (`litellm/`)
- **Main entry point**: `litellm/main.py` - Contains core completion() function
- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
- **Type definitions**: `litellm/types/` - Pydantic models and type hints
- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
### Proxy Server (`litellm/proxy/`)
- **Main server**: `proxy_server.py` - FastAPI application
- **Authentication**: `auth/` - API key management, JWT, OAuth2
- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
- **Guardrails**: `guardrails/` - Safety and content filtering hooks
- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
## Key Patterns
### Provider Implementation
- Providers inherit from base classes in `litellm/llms/base.py`
- Each provider has transformation functions for input/output formatting
- Support both sync and async operations
- Handle streaming responses and function calling
### Error Handling
- Provider-specific exceptions mapped to OpenAI-compatible errors
- Fallback logic handled by Router system
- Comprehensive logging through `litellm/_logging.py`
### Configuration
- YAML config files for proxy server (see `proxy/example_config_yaml/`)
- Environment variables for API keys and settings
- Database schema managed via Prisma (`proxy/schema.prisma`)
## Development Notes
### Code Style
- Uses Black formatter, Ruff linter, MyPy type checker
- Pydantic v2 for data validation
- Async/await patterns throughout
- Type hints required for all public APIs
- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
### Testing Strategy
- Unit tests in `tests/test_litellm/`
- Integration tests for each provider in `tests/llm_translation/`
- Proxy tests in `tests/proxy_unit_tests/`
- Load tests in `tests/load_tests/`
### Database Migrations
- Prisma handles schema migrations
- Migration files auto-generated with `prisma migrate dev`
- Always test migrations against both PostgreSQL and SQLite
### Enterprise Features
- Enterprise-specific code in `enterprise/` directory
- Optional features enabled via environment variables
- Separate licensing and authentication for enterprise features

276
CONTRIBUTING.md Normal file
View file

@ -0,0 +1,276 @@
# Contributing to LiteLLM
Thank you for your interest in contributing to LiteLLM! We welcome contributions of all kinds - from bug fixes and documentation improvements to new features and integrations.
## **Checklist before submitting a PR**
Here are the core requirements for any PR submitted to LiteLLM:
- [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
- [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
- [ ] **Ensure your PR passes all checks**:
- [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
- [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
## **Contributor License Agreement (CLA)**
Before contributing code to LiteLLM, you must sign our [Contributor License Agreement (CLA)](https://cla-assistant.io/BerriAI/litellm). This is a legal requirement for all contributions to be merged into the main repository.
**Important:** We strongly recommend reviewing and signing the CLA before starting work on your contribution to avoid any delays in the PR process.
## Quick Start
### 1. Setup Your Local Development Environment
```bash
# Fork the repository on GitHub (click the Fork button at https://github.com/BerriAI/litellm)
# Then clone your fork locally
git clone https://github.com/YOUR_USERNAME/litellm.git
cd litellm
# Create a new branch for your feature
git checkout -b your-feature-branch
# Install development dependencies
make install-dev
# Verify your setup works
make help
```
That's it! Your local development environment is ready.
### 2. Development Workflow
Here's the recommended workflow for making changes:
```bash
# Make your changes to the code
# ...
# Format your code (auto-fixes formatting issues)
make format
# Run all linting checks (matches CI exactly)
make lint
# Run unit tests to ensure nothing is broken
make test-unit
# Commit your changes
git add .
git commit -m "Your descriptive commit message"
# Push and create a PR
git push origin your-feature-branch
```
## Adding Testing
**Adding at least 1 test is a hard requirement for all PRs.**
### Where to Add Tests
Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm).
- This directory mirrors the structure of the `litellm/` directory
- **Only add mocked tests** - no real LLM API calls in this directory
- For integration tests with real APIs, use the appropriate test directories
### File Naming Convention
The `tests/test_litellm/` directory follows the same structure as `litellm/`:
- `litellm/proxy/caching_routes.py` → `tests/test_litellm/proxy/test_caching_routes.py`
- `litellm/utils.py` → `tests/test_litellm/test_utils.py`
### Example Test
```python
import pytest
from litellm import completion
def test_your_feature():
"""Test your feature with a descriptive docstring."""
# Arrange
messages = [{"role": "user", "content": "Hello"}]
# Act
# Use mocked responses, not real API calls
# Assert
assert expected_result == actual_result
```
## Running Tests and Checks
### Running Unit Tests
Run all unit tests (uses parallel execution for speed):
```bash
make test-unit
```
Run specific test files:
```bash
poetry run pytest tests/test_litellm/test_your_file.py -v
```
### Running Linting and Formatting Checks
Run all linting checks (matches CI exactly):
```bash
make lint
```
Individual linting commands:
```bash
make format-check # Check Black formatting
make lint-ruff # Run Ruff linting
make lint-mypy # Run MyPy type checking
make check-circular-imports # Check for circular imports
make check-import-safety # Check import safety
```
Apply formatting (auto-fixes issues):
```bash
make format
```
### CI Compatibility
To ensure your changes will pass CI, run the exact same checks locally:
```bash
# This runs the same checks as the GitHub workflows
make lint
make test-unit
```
For exact CI compatibility (pins OpenAI version like CI):
```bash
make install-dev-ci # Installs exact CI dependencies
```
## Available Make Commands
Run `make help` to see all available commands:
```bash
make help # Show all available commands
make install-dev # Install development dependencies
make install-proxy-dev # Install proxy development dependencies
make install-test-deps # Install test dependencies (for running tests)
make format # Apply Black code formatting
make format-check # Check Black formatting (matches CI)
make lint # Run all linting checks
make test-unit # Run unit tests
make test-integration # Run integration tests
make test-unit-helm # Run Helm unit tests
```
## Code Quality Standards
LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
Our automated quality checks include:
- **Black** for consistent code formatting
- **Ruff** for linting and code quality
- **MyPy** for static type checking
- **Circular import detection**
- **Import safety validation**
All checks must pass before your PR can be merged.
## Common Issues and Solutions
### 1. Linting Failures
If `make lint` fails:
1. **Formatting issues**: Run `make format` to auto-fix
2. **Ruff issues**: Check the output and fix manually
3. **MyPy issues**: Add proper type hints
4. **Circular imports**: Refactor import dependencies
5. **Import safety**: Fix any unprotected imports
### 2. Test Failures
If `make test-unit` fails:
1. Check if you broke existing functionality
2. Add tests for your new code
3. Ensure tests use mocks, not real API calls
4. Check test file naming conventions
### 3. Common Development Tips
- **Use type hints**: MyPy requires proper type annotations
- **Write descriptive commit messages**: Help reviewers understand your changes
- **Keep PRs focused**: One feature/fix per PR
- **Test edge cases**: Don't just test the happy path
- **Update documentation**: If you change APIs, update docs
## Building and Running Locally
### LiteLLM Proxy Server
To run the proxy server locally:
```bash
# Install proxy dependencies
make install-proxy-dev
# Start the proxy server
poetry run litellm --config your_config.yaml
```
### Docker Development
If you want to build the Docker image yourself:
```bash
# Build using the non-root Dockerfile
docker build -f docker/Dockerfile.non_root -t litellm_dev .
# Run with your config
docker run \
-v $(pwd)/proxy_config.yaml:/app/config.yaml \
-e LITELLM_MASTER_KEY="sk-1234" \
-p 4000:4000 \
litellm_dev \
--config /app/config.yaml --detailed_debug
```
## Submitting Your PR
1. **Push your branch**: `git push origin your-feature-branch`
2. **Create a PR**: Go to GitHub and create a pull request
3. **Fill out the PR template**: Provide clear description of changes
4. **Wait for review**: Maintainers will review and provide feedback
5. **Address feedback**: Make requested changes and push updates
6. **Merge**: Once approved, your PR will be merged!
## Getting Help
If you need help:
- 💬 [Join our Discord](https://discord.gg/wuPM9dRgDw)
- 💬 [Join our Slack](https://www.litellm.ai/support)
- 📧 Email us: ishaan@berri.ai / krrish@berri.ai
- 🐛 [Create an issue](https://github.com/BerriAI/litellm/issues/new)
## What to Contribute
Looking for ideas? Check out:
- 🐛 [Good first issues](https://github.com/BerriAI/litellm/labels/good%20first%20issue)
- 🚀 [Feature requests](https://github.com/BerriAI/litellm/labels/enhancement)
- 📚 Documentation improvements
- 🧪 Test coverage improvements
- 🔌 New LLM provider integrations
Thank you for contributing to LiteLLM! 🚀

60
Dockerfile
View file

@ -1,27 +1,27 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.11.8-slim
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.11.8-slim
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
# Builder stage
FROM $LITELLM_BUILD_IMAGE as builder
FROM $LITELLM_BUILD_IMAGE AS builder
# Set the working directory to /app
WORKDIR /app
# Install build dependencies
RUN apt-get clean && apt-get update && \
apt-get install -y gcc python3-dev && \
rm -rf /var/lib/apt/lists/*
USER root
RUN pip install --upgrade pip && \
pip install build
# Install build dependencies
RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev
RUN python -m pip install build
# Copy the current directory contents into the container at /app
COPY . .
# Build Admin UI
RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Build the package
RUN rm -rf dist/* && python -m build
@ -35,19 +35,20 @@ RUN pip install dist/*.whl
# install dependencies as wheels
RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
# install semantic-cache [Experimental]- we need this here and not in requirements.txt because redisvl pins to pydantic 1.0
RUN pip install redisvl==0.0.7 --no-deps
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT --no-cache-dir
# Build Admin UI
RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh
RUN pip install PyJWT==2.9.0 --no-cache-dir
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE as runtime
FROM $LITELLM_RUNTIME_IMAGE AS runtime
# Ensure runtime stage runs as root
USER root
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@latest
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -61,13 +62,26 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Generate prisma client
RUN prisma generate
RUN chmod +x entrypoint.sh
# Remove test files and keys from dependencies
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
# Install semantic_router and aurelio-sdk using script
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Generate prisma client using the correct schema
RUN prisma generate --schema=./litellm/proxy/schema.prisma
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
ENTRYPOINT ["litellm"]
RUN apk add --no-cache supervisor
COPY docker/supervisord.conf /etc/supervisord.conf
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
ENTRYPOINT ["docker/prod_entrypoint.sh"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
CMD ["--port", "4000"]

52
Dockerfile.alpine
View file

@ -1,52 +0,0 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.11-alpine
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine
# Builder stage
FROM $LITELLM_BUILD_IMAGE as builder
# Set the working directory to /app
WORKDIR /app
# Install build dependencies
RUN apk update && \
apk add --no-cache gcc python3-dev musl-dev && \
rm -rf /var/cache/apk/*
RUN pip install --upgrade pip && \
pip install build
# Copy the current directory contents into the container at /app
COPY . .
# Build the package
RUN rm -rf dist/* && python -m build
# There should be only one wheel file now, assume the build only creates one
RUN ls -1 dist/*.whl | head -1
# Install the package
RUN pip install dist/*.whl
# install dependencies as wheels
RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE as runtime
WORKDIR /app
# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
EXPOSE 4000/tcp
# Set your entrypoint and command
ENTRYPOINT ["litellm"]
CMD ["--port", "4000"]

76
Dockerfile.database
View file

@ -1,76 +0,0 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.11.8-slim
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.11.8-slim
# Builder stage
FROM $LITELLM_BUILD_IMAGE as builder
# Set the working directory to /app
WORKDIR /app
# Install build dependencies
RUN apt-get clean && apt-get update && \
apt-get install -y gcc python3-dev && \
rm -rf /var/lib/apt/lists/*
RUN pip install --upgrade pip && \
pip install build
# Copy the current directory contents into the container at /app
COPY . .
# Build Admin UI
RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh
# Build the package
RUN rm -rf dist/* && python -m build
# There should be only one wheel file now, assume the build only creates one
RUN ls -1 dist/*.whl | head -1
# Install the package
RUN pip install dist/*.whl
# install dependencies as wheels
RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE as runtime
WORKDIR /app
# Copy the current directory contents into the container at /app
COPY . .
RUN ls -la /app
# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# install semantic-cache [Experimental]- we need this here and not in requirements.txt because redisvl pins to pydantic 1.0
RUN pip install redisvl==0.0.7 --no-deps
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT --no-cache-dir
# Build Admin UI
RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh
# Generate prisma client
RUN prisma generate
RUN chmod +x entrypoint.sh
EXPOSE 4000/tcp
# # Set your entrypoint and command
ENTRYPOINT ["litellm"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
# CMD ["--port", "4000", "--detailed_debug"]
CMD ["--port", "4000"]

108
GEMINI.md Normal file
View file

@ -0,0 +1,108 @@
# GEMINI.md
This file provides guidance to Gemini when working with code in this repository.
## Development Commands
### Installation
- `make install-dev` - Install core development dependencies
- `make install-proxy-dev` - Install proxy development dependencies with full feature set
- `make install-test-deps` - Install all test dependencies
### Testing
- `make test` - Run all tests
- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
- `make test-integration` - Run integration tests (excludes unit tests)
- `pytest tests/` - Direct pytest execution
### Code Quality
- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
- `make format` - Apply Black code formatting
- `make lint-ruff` - Run Ruff linting only
- `make lint-mypy` - Run MyPy type checking only
### Single Test Files
- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
### Running Scripts
- `poetry run python script.py` - Run Python scripts (use for non-test files)
### GitHub Issue & PR Templates
When contributing to the project, use the appropriate templates:
**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
- Describe what happened vs. what you expected
- Include relevant log output
- Specify your LiteLLM version
**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
- Describe the feature clearly
- Explain the motivation and use case
**Pull Requests** (`.github/pull_request_template.md`):
- Add at least 1 test in `tests/litellm/`
- Ensure `make test-unit` passes
## Architecture Overview
LiteLLM is a unified interface for 100+ LLM providers with two main components:
### Core Library (`litellm/`)
- **Main entry point**: `litellm/main.py` - Contains core completion() function
- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
- **Type definitions**: `litellm/types/` - Pydantic models and type hints
- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
### Proxy Server (`litellm/proxy/`)
- **Main server**: `proxy_server.py` - FastAPI application
- **Authentication**: `auth/` - API key management, JWT, OAuth2
- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
- **Guardrails**: `guardrails/` - Safety and content filtering hooks
- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
## Key Patterns
### Provider Implementation
- Providers inherit from base classes in `litellm/llms/base.py`
- Each provider has transformation functions for input/output formatting
- Support both sync and async operations
- Handle streaming responses and function calling
### Error Handling
- Provider-specific exceptions mapped to OpenAI-compatible errors
- Fallback logic handled by Router system
- Comprehensive logging through `litellm/_logging.py`
### Configuration
- YAML config files for proxy server (see `proxy/example_config_yaml/`)
- Environment variables for API keys and settings
- Database schema managed via Prisma (`proxy/schema.prisma`)
## Development Notes
### Code Style
- Uses Black formatter, Ruff linter, MyPy type checker
- Pydantic v2 for data validation
- Async/await patterns throughout
- Type hints required for all public APIs
### Testing Strategy
- Unit tests in `tests/test_litellm/`
- Integration tests for each provider in `tests/llm_translation/`
- Proxy tests in `tests/proxy_unit_tests/`
- Load tests in `tests/load_tests/`
### Database Migrations
- Prisma handles schema migrations
- Migration files auto-generated with `prisma migrate dev`
- Always test migrations against both PostgreSQL and SQLite
### Enterprise Features
- Enterprise-specific code in `enterprise/` directory
- Optional features enabled via environment variables
- Separate licensing and authentication for enterprise features

149
Makefile Normal file
View file

@ -0,0 +1,149 @@
# LiteLLM Makefile
# Simple Makefile for running tests and basic development tasks
.PHONY: help test test-unit test-integration test-unit-helm \
info lint lint-dev format \
install-dev install-proxy-dev install-test-deps \
install-helm-unittest check-circular-imports check-import-safety
# Default target
help:
@echo "Available commands:"
@echo " make install-dev - Install development dependencies"
@echo " make install-proxy-dev - Install proxy development dependencies"
@echo " make install-dev-ci - Install dev dependencies (CI-compatible, pins OpenAI)"
@echo " make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)"
@echo " make install-test-deps - Install test dependencies"
@echo " make install-helm-unittest - Install helm unittest plugin"
@echo " make format - Apply Black code formatting"
@echo " make format-check - Check Black code formatting (matches CI)"
@echo " make lint - Run all linting (Ruff, MyPy, Black check, circular imports, import safety)"
@echo " make lint-ruff - Run Ruff linting only"
@echo " make lint-mypy - Run MyPy type checking only"
@echo " make lint-black - Check Black formatting (matches CI)"
@echo " make check-circular-imports - Check for circular imports"
@echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
@echo " make test-unit - Run unit tests (tests/test_litellm)"
@echo " make test-integration - Run integration tests"
@echo " make test-unit-helm - Run helm unit tests"
# Keep PIP simple for edge cases:
PIP := $(shell command -v pip > /dev/null 2>&1 && echo "pip" || echo "python3 -m pip")
# Show info
info:
@echo "PIP: $(PIP)"
# Installation targets
install-dev:
poetry install --with dev
install-proxy-dev:
poetry install --with dev,proxy-dev --extras proxy
# CI-compatible installations (matches GitHub workflows exactly)
install-dev-ci:
$(PIP) install openai==2.8.0
poetry install --with dev
$(PIP) install openai==2.8.0
install-proxy-dev-ci:
poetry install --with dev,proxy-dev --extras proxy
$(PIP) install openai==2.8.0
install-test-deps: install-proxy-dev
poetry run $(PIP) install "pytest-retry==1.6.3"
poetry run $(PIP) install pytest-xdist
poetry run $(PIP) install openapi-core
cd enterprise && poetry run $(PIP) install -e . && cd ..
install-helm-unittest:
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
# Formatting
format: install-dev
cd litellm && poetry run black . && cd ..
format-check: install-dev
cd litellm && poetry run black --check . && cd ..
# Linting targets
lint-ruff: install-dev
cd litellm && poetry run ruff check . && cd ..
# faster linter for developing ...
# inspiration from:
# https://github.com/astral-sh/ruff/discussions/10977
# https://github.com/astral-sh/ruff/discussions/4049
lint-format-changed: install-dev
@git diff origin/main --unified=0 --no-color -- '*.py' | \
perl -ne '\
if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
print "$$file:$$start:1-$$end:999\n"; \
}' | \
while read range; do \
file="$${range%%:*}"; \
lines="$${range#*:}"; \
echo "Formatting $$file (lines $$lines)"; \
poetry run ruff format --range "$$lines" "$$file"; \
done
lint-ruff-dev: install-dev
@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
cd litellm && \
(poetry run ruff check . --output-format=pylint || true) > "$$tmpfile" && \
poetry run diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
cd .. ; \
rm -f "$$tmpfile"
lint-ruff-FULL-dev: install-dev
@files=$$(git diff --name-only origin/main -- '*.py'); \
if [ -n "$$files" ]; then echo "$$files" | xargs poetry run ruff check; \
else echo "No changed .py files to check."; fi
lint-mypy: install-dev
poetry run $(PIP) install types-requests types-setuptools types-redis types-PyYAML
cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
lint-black: format-check
check-circular-imports: install-dev
cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
check-import-safety: install-dev
@poetry run python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Combined linting (matches test-linting.yml workflow)
lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
# Testing targets
test:
poetry run pytest tests/
test-unit: install-test-deps
poetry run pytest tests/test_litellm -x -vv -n 4
test-integration:
poetry run pytest tests/ -k "not test_litellm"
test-unit-helm: install-helm-unittest
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
# LLM Translation testing targets
test-llm-translation: install-test-deps
@echo "Running LLM translation tests..."
@python .github/workflows/run_llm_translation_tests.py
test-llm-translation-single: install-test-deps
@echo "Running single LLM translation test file..."
@if [ -z "$(FILE)" ]; then echo "Usage: make test-llm-translation-single FILE=test_filename.py"; exit 1; fi
@mkdir -p test-results
poetry run pytest tests/llm_translation/$(FILE) \
--junitxml=test-results/junit.xml \
-v --tb=short --maxfail=100 --timeout=300

566
README.md
View file

@ -2,17 +2,20 @@
🚅 LiteLLM
</h1>
<p align="center">
<p align="center">Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, etc.]
<br>
<p align="center">Call 100+ LLMs in OpenAI format. [Bedrock, Azure, OpenAI, VertexAI, Anthropic, Groq, etc.]
</p>
<p align="center">
<a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
<a href="https://railway.app/template/HLP0Ub?referralCode=jch2ME">
<img src="https://railway.app/button.svg" alt="Deploy on Railway">
</a>
</p>
</p>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">OpenAI Proxy Server</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
<h4 align="center">
<a href="https://pypi.org/project/litellm/" target="_blank">
<img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
</a>
<a href="https://dl.circleci.com/status-badge/redirect/gh/BerriAI/litellm/tree/main" target="_blank">
<img src="https://dl.circleci.com/status-badge/img/gh/BerriAI/litellm/tree/main.svg?style=svg" alt="CircleCI">
</a>
<a href="https://www.ycombinator.com/companies/berriai">
<img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23">
</a>
@ -22,30 +25,22 @@
<a href="https://discord.gg/wuPM9dRgDw">
<img src="https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square" alt="Discord">
</a>
<a href="https://www.litellm.ai/support">
<img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
</a>
</h4>
LiteLLM manages:
<img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
- Set Budgets & Rate limits per project, api key, model [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy)
[**Jump to OpenAI Proxy Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) <br>
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
## Use LiteLLM for
🚨 **Stable Release:** Use docker images with: `main-stable` tag. These run through 12 hr load tests (1k req./min).
<details open>
<summary><b>LLMs</b> - Call 100+ LLMs (Python SDK + AI Gateway)</summary>
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
[**All Supported Endpoints**](https://docs.litellm.ai/docs/supported_endpoints) - `/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, `/rerank`, `/a2a`, `/messages` and more.
# Usage ([**Docs**](https://docs.litellm.ai/docs/))
> [!IMPORTANT]
> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
</a>
### Python SDK
```shell
pip install litellm
@ -55,218 +50,358 @@ pip install litellm
from litellm import completion
import os
## set ENV variables
os.environ["OPENAI_API_KEY"] = "your-openai-key"
os.environ["COHERE_API_KEY"] = "your-cohere-key"
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
messages = [{ "content": "Hello, how are you?","role": "user"}]
# OpenAI
response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hello!"}])
# openai call
response = completion(model="gpt-3.5-turbo", messages=messages)
# cohere call
response = completion(model="command-nightly", messages=messages)
print(response)
# Anthropic
response = completion(model="anthropic/claude-sonnet-4-20250514", messages=[{"role": "user", "content": "Hello!"}])
```
Call any model supported by a provider, with `model=<provider_name>/<model_name>`. There might be provider-specific details here, so refer to [provider docs for more information](https://docs.litellm.ai/docs/providers)
### AI Gateway (Proxy Server)
## Async ([Docs](https://docs.litellm.ai/docs/completion/stream#async-completion))
```python
from litellm import acompletion
import asyncio
async def test_get_response():
user_message = "Hello, how are you?"
messages = [{"content": user_message, "role": "user"}]
response = await acompletion(model="gpt-3.5-turbo", messages=messages)
return response
response = asyncio.run(test_get_response())
print(response)
```
## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
```python
from litellm import completion
response = completion(model="gpt-3.5-turbo", messages=messages, stream=True)
for part in response:
print(part.choices[0].delta.content or "")
# claude 2
response = completion('claude-2', messages, stream=True)
for part in response:
print(part.choices[0].delta.content or "")
```
## Logging Observability ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack
```python
from litellm import completion
## set env variables for logging tools
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
os.environ["LANGFUSE_SECRET_KEY"] = ""
os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
os.environ["OPENAI_API_KEY"]
# set callbacks
litellm.success_callback = ["lunary", "langfuse", "athina"] # log input/output to lunary, langfuse, supabase, athina etc
#openai call
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
```
# OpenAI Proxy - ([Docs](https://docs.litellm.ai/docs/simple_proxy))
Set Budgets & Rate limits across multiple projects
The proxy provides:
1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth)
2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class)
3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend)
4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits)
## 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/)
## Quick Start Proxy - CLI
[**Getting Started - E2E Tutorial**](https://docs.litellm.ai/docs/proxy/docker_quick_start) - Setup virtual keys, make your first request
```shell
pip install 'litellm[proxy]'
litellm --model gpt-4o
```
### Step 1: Start litellm proxy
```shell
$ litellm --model huggingface/bigcode/starcoder
#INFO: Proxy running on http://0.0.0.0:4000
```
### Step 2: Make ChatCompletions Request to Proxy
```python
import openai # openai v1.0.0+
client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
])
import openai
print(response)
client = openai.OpenAI(api_key="anything", base_url="http://0.0.0.0:4000")
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello!"}]
)
```
## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys))
[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
UI on `/ui` on your proxy server
![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033)
</details>
Set budgets and rate limits across multiple projects
`POST /key/generate`
<details>
<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
### Request
[**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
```shell
curl 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data-raw '{"models": ["gpt-3.5-turbo", "gpt-4", "claude-2"], "duration": "20m","metadata": {"user": "ishaan@berri.ai", "team": "core-infra"}}'
### Python SDK - A2A Protocol
```python
from litellm.a2a_protocol import A2AClient
from a2a.types import SendMessageRequest, MessageSendParams
from uuid import uuid4
client = A2AClient(base_url="http://localhost:10001")
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello!"}],
"messageId": uuid4().hex,
}
)
)
response = await client.send_message(request)
```
### Expected Response
### AI Gateway (Proxy Server)
```shell
**Step 1.** [Add your Agent to the AI Gateway](https://docs.litellm.ai/docs/a2a#adding-your-agent)
**Step 2.** Call Agent via A2A SDK
```python
from a2a.client import A2ACardResolver, A2AClient
from a2a.types import MessageSendParams, SendMessageRequest
from uuid import uuid4
import httpx
base_url = "http://localhost:4000/a2a/my-agent" # LiteLLM proxy + agent name
headers = {"Authorization": "Bearer sk-1234"} # LiteLLM Virtual Key
async with httpx.AsyncClient(headers=headers) as httpx_client:
resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
agent_card = await resolver.get_agent_card()
client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello!"}],
"messageId": uuid4().hex,
}
)
)
response = await client.send_message(request)
```
[**Docs: A2A Agent Gateway**](https://docs.litellm.ai/docs/a2a)
</details>
<details>
<summary><b>MCP Tools</b> - Connect MCP servers to any LLM (Python SDK + AI Gateway)</summary>
### Python SDK - MCP Bridge
```python
from mcp import ClientSession, StdioServerParameters
from mcp.client.stdio import stdio_client
from litellm import experimental_mcp_client
import litellm
server_params = StdioServerParameters(command="python", args=["mcp_server.py"])
async with stdio_client(server_params) as (read, write):
async with ClientSession(read, write) as session:
await session.initialize()
# Load MCP tools in OpenAI format
tools = await experimental_mcp_client.load_mcp_tools(session=session, format="openai")
# Use with any LiteLLM model
response = await litellm.acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": "What's 3 + 5?"}],
tools=tools
)
```
### AI Gateway - MCP Gateway
**Step 1.** [Add your MCP Server to the AI Gateway](https://docs.litellm.ai/docs/mcp#adding-your-mcp)
**Step 2.** Call MCP tools via `/chat/completions`
```bash
curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Summarize the latest open PR"}],
"tools": [{
"type": "mcp",
"server_url": "litellm_proxy/mcp/github",
"server_label": "github_mcp",
"require_approval": "never"
}]
}'
```
### Use with Cursor IDE
```json
{
"key": "sk-kdEXbIqZRwEeEiHwdg7sFA", # Bearer token
"expires": "2023-11-19T01:38:25.838000+00:00" # datetime object
"mcpServers": {
"LiteLLM": {
"url": "http://localhost:4000/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
}
}
}
```
## Supported Providers ([Docs](https://docs.litellm.ai/docs/providers))
[**Docs: MCP Gateway**](https://docs.litellm.ai/docs/mcp)
| Provider | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) | [Async Image Generation](https://docs.litellm.ai/docs/image_generation) |
| ----------------------------------------------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- |
| [openai](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [azure](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [aws - bedrock](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [google - vertex_ai [Gemini]](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ |
| [google - palm](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | ✅ |
| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | |
| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ |
| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ |
| [huggingface](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [replicate](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | ✅ |
| [together_ai](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | ✅ |
| [openrouter](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | ✅ |
| [ai21](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | ✅ |
| [baseten](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | ✅ |
| [vllm](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | ✅ |
| [nlp_cloud](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | ✅ |
| [aleph alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | ✅ |
| [petals](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | ✅ |
| [ollama](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ |
| [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ |
| [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ |
| [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ |
| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ |
| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ |
</details>
---
## How to use LiteLLM
You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs:
<table style={{width: '100%', tableLayout: 'fixed'}}>
<thead>
<tr>
<th style={{width: '14%'}}></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
</tr>
</thead>
<tbody>
<tr>
<td style={{width: '14%'}}><strong>Use Case</strong></td>
<td style={{width: '43%'}}>Central service (LLM Gateway) to access multiple LLMs</td>
<td style={{width: '43%'}}>Use LiteLLM directly in your Python code</td>
</tr>
<tr>
<td style={{width: '14%'}}><strong>Who Uses It?</strong></td>
<td style={{width: '43%'}}>Gen AI Enablement / ML Platform Teams</td>
<td style={{width: '43%'}}>Developers building LLM projects</td>
</tr>
<tr>
<td style={{width: '14%'}}><strong>Key Features</strong></td>
<td style={{width: '43%'}}>Centralized API gateway with authentication and authorization, multi-tenant cost tracking and spend management per project/user, per-project customization (logging, guardrails, caching), virtual keys for secure access control, admin dashboard UI for monitoring and management</td>
<td style={{width: '43%'}}>Direct Python library integration in your codebase, Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - <a href="https://docs.litellm.ai/docs/routing">Router</a>, application-level load balancing and cost tracking, exception handling with OpenAI-compatible errors, observability callbacks (Lunary, MLflow, Langfuse, etc.)</td>
</tr>
</tbody>
</table>
LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://docs.litellm.ai/docs/simple_proxy) <br>
[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
**Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
## OSS Adopters
<table>
<tr>
<td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
<td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
<td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
<td><h2>Netflix</h2></td>
<td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
</tr>
</table>
## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
|-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ | | | | | | | | | |
| [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [Aleph Alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | | | | | | | |
| [Amazon Nova](https://docs.litellm.ai/docs/providers/amazon_nova) | ✅ | ✅ | ✅ | | | | | | | |
| [Anthropic (`anthropic`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
| [Anthropic Text (`anthropic_text`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
| [Anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | | | | | | | |
| [AssemblyAI (`assemblyai`)](https://docs.litellm.ai/docs/pass_through/assembly_ai) | ✅ | ✅ | ✅ | | | ✅ | | | | |
| [Auto Router (`auto_router`)](https://docs.litellm.ai/docs/proxy/auto_routing) | ✅ | ✅ | ✅ | | | | | | | |
| [AWS - Bedrock (`bedrock`)](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [AWS - Sagemaker (`sagemaker`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Azure (`azure`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [Azure AI (`azure_ai`)](https://docs.litellm.ai/docs/providers/azure_ai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [Azure Text (`azure_text`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Baseten (`baseten`)](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | | | | | | | |
| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | |
| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | |
| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | |
| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | |
| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Cohere (`cohere`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [Cohere Chat (`cohere_chat`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | | | | | | | |
| [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | | | | | | | |
| [Databricks (`databricks`)](https://docs.litellm.ai/docs/providers/databricks) | ✅ | ✅ | ✅ | | | | | | | |
| [DataRobot (`datarobot`)](https://docs.litellm.ai/docs/providers/datarobot) | ✅ | ✅ | ✅ | | | | | | | |
| [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ | | | ✅ | | | | |
| [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | | | | | | | |
| [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | | | | | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [FriendliAI (`friendliai`)](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | | | | | | | |
| [Galadriel (`galadriel`)](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | | | | | | | |
| [GitHub Copilot (`github_copilot`)](https://docs.litellm.ai/docs/providers/github_copilot) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [GitHub Models (`github`)](https://docs.litellm.ai/docs/providers/github) | ✅ | ✅ | ✅ | | | | | | | |
| [Google - PaLM](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | | | | | | | |
| [Google - Vertex AI (`vertex_ai`)](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [Google AI Studio - Gemini (`gemini`)](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | | | | | | | |
| [GradientAI (`gradient_ai`)](https://docs.litellm.ai/docs/providers/gradient_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Groq AI (`groq`)](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | | | | | | | |
| [Heroku (`heroku`)](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | ✅ | | | | | | | |
| [Hosted VLLM (`hosted_vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |
| [Huggingface (`huggingface`)](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [Hyperbolic (`hyperbolic`)](https://docs.litellm.ai/docs/providers/hyperbolic) | ✅ | ✅ | ✅ | | | | | | | |
| [IBM - Watsonx.ai (`watsonx`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Infinity (`infinity`)](https://docs.litellm.ai/docs/providers/infinity) | | | | ✅ | | | | | | |
| [Jina AI (`jina_ai`)](https://docs.litellm.ai/docs/providers/jina_ai) | | | | ✅ | | | | | | |
| [Lambda AI (`lambda_ai`)](https://docs.litellm.ai/docs/providers/lambda_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Lemonade (`lemonade`)](https://docs.litellm.ai/docs/providers/lemonade) | ✅ | ✅ | ✅ | | | | | | | |
| [LiteLLM Proxy (`litellm_proxy`)](https://docs.litellm.ai/docs/providers/litellm_proxy) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [Llamafile (`llamafile`)](https://docs.litellm.ai/docs/providers/llamafile) | ✅ | ✅ | ✅ | | | | | | | |
| [LM Studio (`lm_studio`)](https://docs.litellm.ai/docs/providers/lm_studio) | ✅ | ✅ | ✅ | | | | | | | |
| [Maritalk (`maritalk`)](https://docs.litellm.ai/docs/providers/maritalk) | ✅ | ✅ | ✅ | | | | | | | |
| [Meta - Llama API (`meta_llama`)](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ | | | | | | | |
| [Mistral AI API (`mistral`)](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | |
| [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | |
| [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | | | | | | | |
| [Nscale (`nscale`)](https://docs.litellm.ai/docs/providers/nscale) | ✅ | ✅ | ✅ | | | | | | | |
| [Nvidia NIM (`nvidia_nim`)](https://docs.litellm.ai/docs/providers/nvidia_nim) | ✅ | ✅ | ✅ | | | | | | | |
| [OCI (`oci`)](https://docs.litellm.ai/docs/providers/oci) | ✅ | ✅ | ✅ | | | | | | | |
| [Ollama (`ollama`)](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Ollama Chat (`ollama_chat`)](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | | | | | | | |
| [Oobabooga (`oobabooga`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [OpenAI (`openai`)](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) | | | | ✅ | | | | | | |
| [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | | | | | | | |
| [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | | | | | | | |
| [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | | | | | | | |
| [Predibase (`predibase`)](https://docs.litellm.ai/docs/providers/predibase) | ✅ | ✅ | ✅ | | | | | | | |
| [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) | | | | | ✅ | | | | | |
| [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | | | | | | | |
| [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | |
| [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | |
| [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | |
| [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | | | | | | | |
| [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ | | | | | | | |
| [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ | | | | | | | |
| [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ | | | | | | | |
| [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
| [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |
| [Volcengine (`volcengine`)](https://docs.litellm.ai/docs/providers/volcano) | ✅ | ✅ | ✅ | | | | | | | |
| [Voyage AI (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | |
| [WandB Inference (`wandb`)](https://docs.litellm.ai/docs/providers/wandb_inference) | ✅ | ✅ | ✅ | | | | | | | |
| [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | |
| [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | |
| [Xinference (`xinference`)](https://docs.litellm.ai/docs/providers/xinference) | | | | ✅ | | | | | | |
[**Read the Docs**](https://docs.litellm.ai/docs/)
## Contributing
## Run in Developer mode
### Services
1. Setup .env file in root
2. Run dependant services `docker-compose up db prometheus`
To contribute: Clone the repo locally -> Make a change -> Submit a PR with the change.
### Backend
1. (In root) create virtual environment `python -m venv .venv`
2. Activate virtual environment `source .venv/bin/activate`
3. Install dependencies `pip install -e ".[all]"`
4. `pip install prisma`
5. `prisma generate`
6. Start proxy backend `python litellm/proxy/proxy_cli.py`
Here's how to modify the repo locally:
Step 1: Clone the repo
```
git clone https://github.com/BerriAI/litellm.git
```
Step 2: Navigate into the project, and install dependencies:
```
cd litellm
poetry install
```
Step 3: Test your change:
```
cd litellm/tests # pwd: Documents/litellm/litellm/tests
poetry run flake8
poetry run pytest .
```
Step 4: Submit a PR with your changes! 🚀
- push your fork to your GitHub repo
- submit a PR from there
### Frontend
1. Navigate to `ui/litellm-dashboard`
2. Install dependencies `npm install`
3. Run `npm run dev` to start the dashboard
# Enterprise
For companies that need better security, user management and professional support
[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
This covers:
This covers:
- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
- ✅ **Feature Prioritization**
- ✅ **Custom Integrations**
@ -274,10 +409,46 @@ This covers:
- ✅ **Custom SLAs**
- ✅ **Secure access with Single Sign-On**
# Contributing
We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features, or improving documentation, we appreciate your help.
## Quick Start for Contributors
This requires poetry to be installed.
```bash
git clone https://github.com/BerriAI/litellm.git
cd litellm
make install-dev # Install development dependencies
make format # Format your code
make lint # Run all linting checks
make test-unit # Run unit tests
make format-check # Check formatting only
```
For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
## Code Quality / Linting
LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
Our automated checks include:
- **Black** for code formatting
- **Ruff** for linting and code quality
- **MyPy** for type checking
- **Circular import detection**
- **Import safety checks**
All these checks must pass before your PR can be merged.
# Support / talk with founders
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- [Community Slack 💭](https://www.litellm.ai/support)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
@ -299,3 +470,4 @@ This covers:
<a href="https://github.com/BerriAI/litellm/graphs/contributors">
<img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
</a>

3
ci_cd/.grype.yaml Normal file
View file

@ -0,0 +1,3 @@
ignore:
- vulnerability: CVE-2026-22184
reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists

40
ci_cd/TEST_KEY_PATTERNS.md Normal file
View file

@ -0,0 +1,40 @@
# Test Key Patterns Standard
Standard patterns for test/mock keys and credentials in the LiteLLM codebase to avoid triggering secret detection.
## How GitGuardian Works
GitGuardian uses **machine learning and entropy analysis**, not just pattern matching:
- **Low entropy** values (like `sk-1234`, `postgres`) are automatically ignored
- **High entropy** values (realistic-looking secrets) trigger detection
- **Context-aware** detection understands code syntax like `os.environ["KEY"]`
## Recommended Test Key Patterns
### Option 1: Low Entropy Values (Simplest)
These won't trigger GitGuardian's ML detector:
```python
api_key = "sk-1234"
api_key = "sk-12345"
database_password = "postgres"
token = "test123"
```
### Option 2: High Entropy with Test Prefixes
If you need realistic-looking test keys with high entropy, use these prefixes:
```python
api_key = "sk-test-abc123def456ghi789..." # OpenAI-style test key
api_key = "sk-mock-1234567890abcdef1234..." # Mock key
api_key = "sk-fake-xyz789uvw456rst123..." # Fake key
token = "test-api-key-with-high-entropy"
```
## Configured Ignore Patterns
These patterns are in `.gitguardian.yaml` for high-entropy test keys:
- `sk-test-*` - OpenAI-style test keys
- `sk-mock-*` - Mock API keys
- `sk-fake-*` - Fake API keys
- `test-api-key` - Generic test tokens

60
ci_cd/baseline_db.py Normal file
View file

@ -0,0 +1,60 @@
import subprocess
from pathlib import Path
from datetime import datetime
def create_baseline():
"""Create baseline migration in deploy/migrations"""
try:
# Get paths
root_dir = Path(__file__).parent.parent
deploy_dir = root_dir / "deploy"
migrations_dir = deploy_dir / "migrations"
schema_path = root_dir / "schema.prisma"
# Create migrations directory
migrations_dir.mkdir(parents=True, exist_ok=True)
# Create migration_lock.toml if it doesn't exist
lock_file = migrations_dir / "migration_lock.toml"
if not lock_file.exists():
lock_file.write_text('provider = "postgresql"\n')
# Create timestamp-based migration directory
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
migration_dir = migrations_dir / f"{timestamp}_baseline"
migration_dir.mkdir(parents=True, exist_ok=True)
# Generate migration SQL
result = subprocess.run(
[
"prisma",
"migrate",
"diff",
"--from-empty",
"--to-schema-datamodel",
str(schema_path),
"--script",
],
capture_output=True,
text=True,
check=True,
)
# Write the SQL to migration.sql
migration_file = migration_dir / "migration.sql"
migration_file.write_text(result.stdout)
print(f"Created baseline migration in {migration_dir}")
return True
except subprocess.CalledProcessError as e:
print(f"Error running prisma command: {e.stderr}")
return False
except Exception as e:
print(f"Error creating baseline migration: {str(e)}")
return False
if __name__ == "__main__":
create_baseline()

28
ci_cd/check_file_length.py Normal file
View file

@ -0,0 +1,28 @@
import sys
def check_file_length(max_lines, filenames):
bad_files = []
for filename in filenames:
with open(filename, "r") as file:
lines = file.readlines()
if len(lines) > max_lines:
bad_files.append((filename, len(lines)))
return bad_files
if __name__ == "__main__":
max_lines = int(sys.argv[1])
filenames = sys.argv[2:]
bad_files = check_file_length(max_lines, filenames)
if bad_files:
bad_files.sort(
key=lambda x: x[1], reverse=True
) # Sort files by length in descending order
for filename, length in bad_files:
print(f"{filename}: {length} lines")
sys.exit(1)
else:
sys.exit(0)

19
ci_cd/publish-proxy-extras.sh Normal file
View file

@ -0,0 +1,19 @@
#!/bin/bash
# Exit on error
set -e
echo "🚀 Building and publishing litellm-proxy-extras"
# Navigate to litellm-proxy-extras directory
cd "$(dirname "$0")/../litellm-proxy-extras"
# Build the package
echo "📦 Building package..."
poetry build
# Publish to PyPI
echo "🌎 Publishing to PyPI..."
poetry publish
echo "✅ Done! Package published successfully"

95
ci_cd/run_migration.py Normal file
View file

@ -0,0 +1,95 @@
import os
import subprocess
from pathlib import Path
from datetime import datetime
import testing.postgresql
import shutil
def create_migration(migration_name: str = None):
"""
Create a new migration SQL file in the migrations directory by comparing
current database state with schema
Args:
migration_name (str): Name for the migration
"""
try:
# Get paths
root_dir = Path(__file__).parent.parent
migrations_dir = root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
schema_path = root_dir / "schema.prisma"
# Create temporary PostgreSQL database
with testing.postgresql.Postgresql() as postgresql:
db_url = postgresql.url()
# Create temporary migrations directory next to schema.prisma
temp_migrations_dir = schema_path.parent / "migrations"
try:
# Copy existing migrations to temp directory
if temp_migrations_dir.exists():
shutil.rmtree(temp_migrations_dir)
shutil.copytree(migrations_dir, temp_migrations_dir)
# Apply existing migrations to temp database
os.environ["DATABASE_URL"] = db_url
subprocess.run(
["prisma", "migrate", "deploy", "--schema", str(schema_path)],
check=True,
)
# Generate diff between current database and schema
result = subprocess.run(
[
"prisma",
"migrate",
"diff",
"--from-url",
db_url,
"--to-schema-datamodel",
str(schema_path),
"--script",
],
capture_output=True,
text=True,
check=True,
)
if result.stdout.strip():
# Generate timestamp and create migration directory
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
migration_name = migration_name or "unnamed_migration"
migration_dir = migrations_dir / f"{timestamp}_{migration_name}"
migration_dir.mkdir(parents=True, exist_ok=True)
# Write the SQL to migration.sql
migration_file = migration_dir / "migration.sql"
migration_file.write_text(result.stdout)
print(f"Created migration in {migration_dir}")
return True
else:
print("No schema changes detected. Migration not needed.")
return False
finally:
# Clean up: remove temporary migrations directory
if temp_migrations_dir.exists():
shutil.rmtree(temp_migrations_dir)
except subprocess.CalledProcessError as e:
print(f"Error generating migration: {e.stderr}")
return False
except Exception as e:
print(f"Error creating migration: {str(e)}")
return False
if __name__ == "__main__":
# If running directly, can optionally pass migration name as argument
import sys
migration_name = sys.argv[1] if len(sys.argv) > 1 else None
create_migration(migration_name)

251
ci_cd/security_scans.sh Executable file
View file

@ -0,0 +1,251 @@
#!/bin/bash
# Security Scans Script for LiteLLM
# This script runs comprehensive security scans including Trivy and Grype
set -e
echo "Starting security scans for LiteLLM..."
# Function to install Trivy and required tools
install_trivy() {
echo "Installing Trivy and required tools..."
sudo apt-get update
sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
sudo apt-get update
sudo apt-get install trivy
echo "Trivy and required tools installed successfully"
}
# Function to install Grype
install_grype() {
echo "Installing Grype..."
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
echo "Grype installed successfully"
}
# Function to install ggshield
install_ggshield() {
echo "Installing ggshield..."
pip3 install --upgrade pip
pip3 install ggshield
echo "ggshield installed successfully"
}
# # Function to run secret detection scans
# run_secret_detection() {
# echo "Running secret detection scans..."
# if ! command -v ggshield &> /dev/null; then
# install_ggshield
# fi
# # Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
# if [ -z "$GITGUARDIAN_API_KEY" ]; then
# echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
# echo "ggshield requires a GitGuardian API key to scan for secrets."
# echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
# exit 1
# fi
# echo "Scanning codebase for secrets..."
# echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
# echo "ggshield will automatically handle rate limits and retry as needed."
# echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
# # Use --recursive for directory scanning and auto-confirm if prompted
# # .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
# # GITGUARDIAN_API_KEY environment variable will be used for authentication
# echo y | ggshield secret scan path . --recursive || {
# echo ""
# echo "=========================================="
# echo "ERROR: Secret Detection Failed"
# echo "=========================================="
# echo "ggshield has detected secrets in the codebase."
# echo "Please review discovered secrets above, revoke any actively used secrets"
# echo "from underlying systems and make changes to inject secrets dynamically at runtime."
# echo ""
# echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
# echo "=========================================="
# echo ""
# exit 1
# }
# echo "Secret detection scans completed successfully"
# }
# Function to run Trivy scans
run_trivy_scans() {
echo "Running Trivy scans..."
echo "Scanning LiteLLM Docs..."
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
echo "Scanning LiteLLM UI..."
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
echo "Trivy scans completed successfully"
}
# Function to build and scan Docker images with Grype
run_grype_scans() {
echo "Running Grype scans..."
# Temporarily add wheel files to .dockerignore for security scans
echo "Temporarily modifying .dockerignore to exclude problematic wheel files..."
cp .dockerignore .dockerignore.backup 2>/dev/null || touch .dockerignore.backup
echo "/*.whl" >> .dockerignore
# Build and scan Dockerfile.database
echo "Building and scanning Dockerfile.database..."
docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
grype litellm-database:latest --config ci_cd/.grype.yaml --fail-on critical
# Build and scan main Dockerfile
echo "Building and scanning main Dockerfile..."
docker build --no-cache -t litellm:latest .
grype litellm:latest --config ci_cd/.grype.yaml --fail-on critical
# Restore original .dockerignore
echo "Restoring original .dockerignore..."
mv .dockerignore.backup .dockerignore
# Scan the locally built LiteLLM image for vulnerabilities with CVSS >= 4.0
echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
echo "Using locally built image: litellm:latest"
# Allowlist of CVEs to be ignored in failure threshold/reporting
# - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
# - GHSA-4xh5-x5gv-qwph: GitHub Security Advisory alias for CVE-2025-8869
# - GHSA-5j98-mcp5-4vw2: glob CLI command injection via -c/--cmd; glob CLI is not used in the litellm runtime image,
# and the vulnerable versions are pulled in only via OS-level/node tooling outside of our application code
ALLOWED_CVES=(
"CVE-2025-8869"
"GHSA-4xh5-x5gv-qwph"
"CVE-2025-8291" # no fix available as of Oct 11, 2025
"GHSA-5j98-mcp5-4vw2"
"CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
"CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
"CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
"CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet
"CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
"CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
"CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
"CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
"GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
"GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
"GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
"CVE-2025-59465" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-55131" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59466" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-55130" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59467" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2026-21637" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-15281" # No fix available yet
"CVE-2026-0865" # No fix available yet
"CVE-2025-15282" # No fix available yet
"CVE-2026-0672" # No fix available yet
"CVE-2025-15366" # No fix available yet
"CVE-2025-15367" # No fix available yet
"CVE-2025-12781" # No fix available yet
"CVE-2025-11468" # No fix available yet
"CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
)
# Build JSON array of allowlisted CVE IDs for jq
ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .)
echo "Checking for vulnerabilities with CVSS score >= 4.0..."
echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
echo ""
# Show all high-severity vulnerabilities for transparency
TOTAL_HIGH_SEVERITY=$(grype litellm:latest -o json | jq -r '
.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| .vulnerability.id' | wc -l)
if [ "$TOTAL_HIGH_SEVERITY" -gt 0 ]; then
echo "Total vulnerabilities found with CVSS >= 4.0: $TOTAL_HIGH_SEVERITY"
echo ""
echo "All high-severity vulnerabilities (including allowlisted):"
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Allowlisted"],
(.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, (if (.vulnerability.id as $id | $allow | index($id)) then "YES" else "NO" end)])
| @tsv' | column -t -s $'\t'
echo ""
fi
HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| .vulnerability.id' | wc -l)
if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
echo ""
echo "=========================================="
echo "ERROR: Security Scan Failed"
echo "=========================================="
echo "Found $HIGH_SEVERITY_COUNT non-allowlisted vulnerabilities with CVSS score >= 4.0 in litellm:latest"
echo ""
echo "These vulnerabilities are NOT in the allowlist and must be addressed."
echo "Current allowlisted CVEs: ${ALLOWED_CVES[*]}"
echo ""
echo "Detailed vulnerability report:"
echo ""
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
(.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
| @tsv' | column -t -s $'\t'
echo ""
echo "=========================================="
echo "Action Required:"
echo "=========================================="
echo "1. If a fix is available, update the package to the fixed version"
echo "2. If the vulnerability is not applicable or has no fix:"
echo " - Add the CVE/GHSA ID to ALLOWED_CVES array in ci_cd/security_scans.sh"
echo " - Add a comment explaining why it's safe to ignore"
echo ""
echo "Note: Some vulnerabilities may have multiple IDs (CVE-XXXX and GHSA-XXXX)."
echo "Add all relevant IDs to the allowlist if they refer to the same issue."
echo "=========================================="
echo ""
exit 1
else
echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"
fi
echo "Grype scans completed successfully"
}
# Main execution
main() {
echo "Installing security scanning tools..."
install_trivy
install_grype
# echo "Running secret detection scans..."
# run_secret_detection
echo "Running filesystem vulnerability scans..."
run_trivy_scans
echo "Running Docker image vulnerability scans..."
run_grype_scans
echo "All security scans completed successfully!"
}
# Execute main function
main "$@"

9
ci_cd/security_scans_readme.md Normal file
View file

@ -0,0 +1,9 @@
# Security Scans
## Scans that run:
- Trivy scan on `./docs/` (HIGH/CRITICAL/MEDIUM)
- Trivy scan on `./ui/` (HIGH/CRITICAL/MEDIUM)
- Grype scan on `Dockerfile.database` (fails on CRITICAL)
- Grype scan on main `Dockerfile` (fails on CRITICAL)
- Grype CVSS ≥ 4.0 scan on main `Dockerfile` (fails any vulnerabilities with CVSS ≥ 4.0)

32
codecov.yaml Normal file
View file

@ -0,0 +1,32 @@
component_management:
individual_components:
- component_id: "Router"
paths:
- "router"
- component_id: "LLMs"
paths:
- "*/llms/*"
- component_id: "Caching"
paths:
- "*/caching/*"
- ".*redis.*"
- component_id: "litellm_logging"
paths:
- "*/integrations/*"
- ".*litellm_logging.*"
- component_id: "Proxy_Authentication"
paths:
- "*/proxy/auth/**"
comment:
layout: "header, diff, flags, components" # show component info in the PR comment
coverage:
status:
project:
default:
target: auto
threshold: 1% # at maximum allow project coverage to drop by 1%
patch:
default:
target: auto
threshold: 0% # patch coverage should be 100%

1470
cookbook/Benchmarking_LLMs_by_use_case.ipynb vendored
View file

File diff suppressed because one or more lines are too long

1154
cookbook/Evaluating_LLMs.ipynb vendored
View file

File diff suppressed because one or more lines are too long

833
cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb vendored
View file

@ -1,423 +1,422 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "BmX0b5Ueh91v"
},
"source": [
"# LiteLLM - Azure OpenAI + OpenAI Calls\n",
"This notebook covers the following for Azure OpenAI + OpenAI:\n",
"* Completion - Quick start\n",
"* Completion - Streaming\n",
"* Completion - Azure, OpenAI in separate threads\n",
"* Completion - Stress Test 10 requests in parallel\n",
"* Completion - Azure, OpenAI in the same thread"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# LiteLLM - Azure OpenAI + OpenAI Calls\n",
"This notebook covers the following for Azure OpenAI + OpenAI:\n",
"* Completion - Quick start\n",
"* Completion - Streaming\n",
"* Completion - Azure, OpenAI in separate threads\n",
"* Completion - Stress Test 10 requests in parallel\n",
"* Completion - Azure, OpenAI in the same thread"
],
"metadata": {
"id": "BmX0b5Ueh91v"
}
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "iHq4d0dpfawS"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "mnveHO5dfcB0"
},
"outputs": [],
"source": [
"import os"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "eo88QUdbiDIE"
},
"source": [
"## Completion - Quick start"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "5OSosWNCfc_2",
"outputId": "c52344b1-2458-4695-a7eb-a9b076893348"
},
"outputs": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "iHq4d0dpfawS"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"source": [
"import os, litellm"
],
"metadata": {
"id": "mnveHO5dfcB0"
},
"execution_count": 2,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Completion - Quick start"
],
"metadata": {
"id": "eo88QUdbiDIE"
}
},
{
"cell_type": "code",
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"# openai configs\n",
"os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
"\n",
"# azure openai configs\n",
"os.environ[\"AZURE_API_KEY\"] = \"\"\n",
"os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
"os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
"\n",
"\n",
"# openai call\n",
"response = completion(\n",
" model = \"gpt-3.5-turbo\",\n",
" messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(\"Openai Response\\n\")\n",
"print(response)\n",
"\n",
"\n",
"\n",
"# azure call\n",
"response = completion(\n",
" model = \"azure/your-azure-deployment\",\n",
" messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(\"Azure Response\\n\")\n",
"print(response)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "5OSosWNCfc_2",
"outputId": "c52344b1-2458-4695-a7eb-a9b076893348"
},
"execution_count": 12,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Openai Response\n",
"\n",
"{\n",
" \"id\": \"chatcmpl-7yjVOEKCPw2KdkfIaM3Ao1tIXp8EM\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694708958,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"I'm an AI, so I don't have feelings, but I'm here to help you. How can I assist you?\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 13,\n",
" \"completion_tokens\": 26,\n",
" \"total_tokens\": 39\n",
" }\n",
"}\n",
"Azure Response\n",
"\n",
"{\n",
" \"id\": \"chatcmpl-7yjVQ6m2R2HRtnKHRRFp6JzL4Fjez\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694708960,\n",
" \"model\": \"gpt-35-turbo\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"finish_reason\": \"stop\",\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello there! As an AI language model, I don't have feelings but I'm functioning well. How can I assist you today?\"\n",
" }\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"completion_tokens\": 27,\n",
" \"prompt_tokens\": 14,\n",
" \"total_tokens\": 41\n",
" }\n",
"}\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Completion - Streaming"
],
"metadata": {
"id": "dQMkM-diiKdE"
}
},
{
"cell_type": "code",
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"# openai configs\n",
"os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
"\n",
"# azure openai configs\n",
"os.environ[\"AZURE_API_KEY\"] = \"\"\n",
"os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
"os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
"\n",
"\n",
"# openai call\n",
"response = completion(\n",
" model = \"gpt-3.5-turbo\",\n",
" messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" stream=True\n",
")\n",
"print(\"OpenAI Streaming response\")\n",
"for chunk in response:\n",
" print(chunk)\n",
"\n",
"# azure call\n",
"response = completion(\n",
" model = \"azure/your-azure-deployment\",\n",
" messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" stream=True\n",
")\n",
"print(\"Azure Streaming response\")\n",
"for chunk in response:\n",
" print(chunk)\n"
],
"metadata": {
"id": "uVvJDVn4g1i1"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Completion - Azure, OpenAI in separate threads"
],
"metadata": {
"id": "4xrOPnt-oqwm"
}
},
{
"cell_type": "code",
"source": [
"import os\n",
"import threading\n",
"from litellm import completion\n",
"\n",
"# Function to make a completion call\n",
"def make_completion(model, messages):\n",
" response = completion(\n",
" model=model,\n",
" messages=messages\n",
" )\n",
"\n",
" print(f\"Response for {model}: {response}\")\n",
"\n",
"# openai configs\n",
"os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
"\n",
"# azure openai configs\n",
"os.environ[\"AZURE_API_KEY\"] = \"\"\n",
"os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
"os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
"\n",
"# Define the messages for the completions\n",
"messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
"\n",
"# Create threads for making the completions\n",
"thread1 = threading.Thread(target=make_completion, args=(\"gpt-3.5-turbo\", messages))\n",
"thread2 = threading.Thread(target=make_completion, args=(\"azure/your-azure-deployment\", messages))\n",
"\n",
"# Start both threads\n",
"thread1.start()\n",
"thread2.start()\n",
"\n",
"# Wait for both threads to finish\n",
"thread1.join()\n",
"thread2.join()\n",
"\n",
"print(\"Both completions are done.\")"
],
"metadata": {
"id": "V5b5taJPjvC3"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Completion - Stress Test 10 requests in parallel\n",
"\n"
],
"metadata": {
"id": "lx8DbMBqoAoN"
}
},
{
"cell_type": "code",
"source": [
"import os\n",
"import threading\n",
"from litellm import completion\n",
"\n",
"# Function to make a completion call\n",
"def make_completion(model, messages):\n",
" response = completion(\n",
" model=model,\n",
" messages=messages\n",
" )\n",
"\n",
" print(f\"Response for {model}: {response}\")\n",
"\n",
"# Set your API keys\n",
"os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
"os.environ[\"AZURE_API_KEY\"] = \"\"\n",
"os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
"os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
"\n",
"# Define the messages for the completions\n",
"messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
"\n",
"# Create and start 10 threads for making completions\n",
"threads = []\n",
"for i in range(10):\n",
" thread = threading.Thread(target=make_completion, args=(\"gpt-3.5-turbo\" if i % 2 == 0 else \"azure/your-azure-deployment\", messages))\n",
" threads.append(thread)\n",
" thread.start()\n",
"\n",
"# Wait for all threads to finish\n",
"for thread in threads:\n",
" thread.join()\n",
"\n",
"print(\"All completions are done.\")\n"
],
"metadata": {
"id": "pHYANOlOkoDh"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Completion - Azure, OpenAI in the same thread"
],
"metadata": {
"id": "yB2NDOO4oxrp"
}
},
{
"cell_type": "code",
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"# Function to make both OpenAI and Azure completions\n",
"def make_completions():\n",
" # Set your OpenAI API key\n",
" os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
"\n",
" # OpenAI completion\n",
" openai_response = completion(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
" )\n",
"\n",
" print(\"OpenAI Response:\", openai_response)\n",
"\n",
" # Set your Azure OpenAI API key and configuration\n",
" os.environ[\"AZURE_API_KEY\"] = \"\"\n",
" os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
" os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
"\n",
" # Azure OpenAI completion\n",
" azure_response = completion(\n",
" model=\"azure/your-azure-deployment\",\n",
" messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
" )\n",
"\n",
" print(\"Azure OpenAI Response:\", azure_response)\n",
"\n",
"# Call the function to make both completions in one thread\n",
"make_completions()\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "HTBqwzxpnxab",
"outputId": "f3bc0efe-e4d5-44d5-a193-97d178cfbe14"
},
"execution_count": 23,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"OpenAI Response: {\n",
" \"id\": \"chatcmpl-7yjzrDeOeVeSrQ00tApmTxEww3vBS\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694710847,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! I'm an AI, so I don't have feelings, but I'm here to help you. How can I assist you today?\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 13,\n",
" \"completion_tokens\": 29,\n",
" \"total_tokens\": 42\n",
" }\n",
"}\n",
"Azure OpenAI Response: {\n",
" \"id\": \"chatcmpl-7yjztAQ0gK6IMQt7cvLroMSOoXkeu\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694710849,\n",
" \"model\": \"gpt-35-turbo\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"finish_reason\": \"stop\",\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"As an AI language model, I don't have feelings but I'm functioning properly. Thank you for asking! How can I assist you today?\"\n",
" }\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"completion_tokens\": 29,\n",
" \"prompt_tokens\": 14,\n",
" \"total_tokens\": 43\n",
" }\n",
"}\n"
]
}
]
"name": "stdout",
"output_type": "stream",
"text": [
"Openai Response\n",
"\n",
"{\n",
" \"id\": \"chatcmpl-7yjVOEKCPw2KdkfIaM3Ao1tIXp8EM\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694708958,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"I'm an AI, so I don't have feelings, but I'm here to help you. How can I assist you?\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 13,\n",
" \"completion_tokens\": 26,\n",
" \"total_tokens\": 39\n",
" }\n",
"}\n",
"Azure Response\n",
"\n",
"{\n",
" \"id\": \"chatcmpl-7yjVQ6m2R2HRtnKHRRFp6JzL4Fjez\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694708960,\n",
" \"model\": \"gpt-35-turbo\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"finish_reason\": \"stop\",\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello there! As an AI language model, I don't have feelings but I'm functioning well. How can I assist you today?\"\n",
" }\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"completion_tokens\": 27,\n",
" \"prompt_tokens\": 14,\n",
" \"total_tokens\": 41\n",
" }\n",
"}\n"
]
}
]
],
"source": [
"from litellm import completion\n",
"\n",
"# openai configs\n",
"os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
"\n",
"# azure openai configs\n",
"os.environ[\"AZURE_API_KEY\"] = \"\"\n",
"os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
"os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
"\n",
"\n",
"# openai call\n",
"response = completion(\n",
" model = \"gpt-3.5-turbo\",\n",
" messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(\"Openai Response\\n\")\n",
"print(response)\n",
"\n",
"\n",
"\n",
"# azure call\n",
"response = completion(\n",
" model = \"azure/your-azure-deployment\",\n",
" messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(\"Azure Response\\n\")\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "dQMkM-diiKdE"
},
"source": [
"## Completion - Streaming"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "uVvJDVn4g1i1"
},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"# openai configs\n",
"os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
"\n",
"# azure openai configs\n",
"os.environ[\"AZURE_API_KEY\"] = \"\"\n",
"os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
"os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
"\n",
"\n",
"# openai call\n",
"response = completion(\n",
" model = \"gpt-3.5-turbo\",\n",
" messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" stream=True\n",
")\n",
"print(\"OpenAI Streaming response\")\n",
"for chunk in response:\n",
" print(chunk)\n",
"\n",
"# azure call\n",
"response = completion(\n",
" model = \"azure/your-azure-deployment\",\n",
" messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" stream=True\n",
")\n",
"print(\"Azure Streaming response\")\n",
"for chunk in response:\n",
" print(chunk)\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "4xrOPnt-oqwm"
},
"source": [
"## Completion - Azure, OpenAI in separate threads"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "V5b5taJPjvC3"
},
"outputs": [],
"source": [
"import os\n",
"import threading\n",
"from litellm import completion\n",
"\n",
"# Function to make a completion call\n",
"def make_completion(model, messages):\n",
" response = completion(\n",
" model=model,\n",
" messages=messages\n",
" )\n",
"\n",
" print(f\"Response for {model}: {response}\")\n",
"\n",
"# openai configs\n",
"os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
"\n",
"# azure openai configs\n",
"os.environ[\"AZURE_API_KEY\"] = \"\"\n",
"os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
"os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
"\n",
"# Define the messages for the completions\n",
"messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
"\n",
"# Create threads for making the completions\n",
"thread1 = threading.Thread(target=make_completion, args=(\"gpt-3.5-turbo\", messages))\n",
"thread2 = threading.Thread(target=make_completion, args=(\"azure/your-azure-deployment\", messages))\n",
"\n",
"# Start both threads\n",
"thread1.start()\n",
"thread2.start()\n",
"\n",
"# Wait for both threads to finish\n",
"thread1.join()\n",
"thread2.join()\n",
"\n",
"print(\"Both completions are done.\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "lx8DbMBqoAoN"
},
"source": [
"## Completion - Stress Test 10 requests in parallel\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "pHYANOlOkoDh"
},
"outputs": [],
"source": [
"import os\n",
"import threading\n",
"from litellm import completion\n",
"\n",
"# Function to make a completion call\n",
"def make_completion(model, messages):\n",
" response = completion(\n",
" model=model,\n",
" messages=messages\n",
" )\n",
"\n",
" print(f\"Response for {model}: {response}\")\n",
"\n",
"# Set your API keys\n",
"os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
"os.environ[\"AZURE_API_KEY\"] = \"\"\n",
"os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
"os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
"\n",
"# Define the messages for the completions\n",
"messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
"\n",
"# Create and start 10 threads for making completions\n",
"threads = []\n",
"for i in range(10):\n",
" thread = threading.Thread(target=make_completion, args=(\"gpt-3.5-turbo\" if i % 2 == 0 else \"azure/your-azure-deployment\", messages))\n",
" threads.append(thread)\n",
" thread.start()\n",
"\n",
"# Wait for all threads to finish\n",
"for thread in threads:\n",
" thread.join()\n",
"\n",
"print(\"All completions are done.\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "yB2NDOO4oxrp"
},
"source": [
"## Completion - Azure, OpenAI in the same thread"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "HTBqwzxpnxab",
"outputId": "f3bc0efe-e4d5-44d5-a193-97d178cfbe14"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"OpenAI Response: {\n",
" \"id\": \"chatcmpl-7yjzrDeOeVeSrQ00tApmTxEww3vBS\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694710847,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"Hello! I'm an AI, so I don't have feelings, but I'm here to help you. How can I assist you today?\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 13,\n",
" \"completion_tokens\": 29,\n",
" \"total_tokens\": 42\n",
" }\n",
"}\n",
"Azure OpenAI Response: {\n",
" \"id\": \"chatcmpl-7yjztAQ0gK6IMQt7cvLroMSOoXkeu\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1694710849,\n",
" \"model\": \"gpt-35-turbo\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"finish_reason\": \"stop\",\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"As an AI language model, I don't have feelings but I'm functioning properly. Thank you for asking! How can I assist you today?\"\n",
" }\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"completion_tokens\": 29,\n",
" \"prompt_tokens\": 14,\n",
" \"total_tokens\": 43\n",
" }\n",
"}\n"
]
}
],
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"# Function to make both OpenAI and Azure completions\n",
"def make_completions():\n",
" # Set your OpenAI API key\n",
" os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
"\n",
" # OpenAI completion\n",
" openai_response = completion(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
" )\n",
"\n",
" print(\"OpenAI Response:\", openai_response)\n",
"\n",
" # Set your Azure OpenAI API key and configuration\n",
" os.environ[\"AZURE_API_KEY\"] = \"\"\n",
" os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n",
" os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n",
"\n",
" # Azure OpenAI completion\n",
" azure_response = completion(\n",
" model=\"azure/your-azure-deployment\",\n",
" messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n",
" )\n",
"\n",
" print(\"Azure OpenAI Response:\", azure_response)\n",
"\n",
"# Call the function to make both completions in one thread\n",
"make_completions()\n"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

474
cookbook/LiteLLM_CometAPI.ipynb vendored Normal file
View file

File diff suppressed because one or more lines are too long

867
cookbook/LiteLLM_Comparing_LLMs.ipynb vendored
View file

File diff suppressed because one or more lines are too long

902
cookbook/LiteLLM_HuggingFace.ipynb vendored
View file

@ -1,28 +1,15 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"source": [
"## LiteLLM HuggingFace\n",
"Docs for huggingface: https://docs.litellm.ai/docs/providers/huggingface"
],
"metadata": {
"id": "9dKM5k8qsMIj"
}
},
"source": [
"## LiteLLM Hugging Face\n",
"\n",
"Docs for huggingface: https://docs.litellm.ai/docs/providers/huggingface\n"
]
},
{
"cell_type": "code",
@ -37,182 +24,22 @@
},
{
"cell_type": "markdown",
"source": [
"## HuggingFace TGI Model - Deployed Inference Endpoints\n",
"Steps to use\n",
"* set `api_base` to your deployed api base\n",
"* Add the `huggingface/` prefix to your model so litellm knows it's a huggingface Deployed Inference Endpoint"
],
"metadata": {
"id": "-klhAhjLtclv"
}
},
{
"cell_type": "code",
"source": [
"import os\n",
"import litellm\n",
"\n",
"os.environ[\"HUGGINGFACE_API_KEY\"] = \"\"\n",
"\n",
"# TGI model: Call https://huggingface.co/glaiveai/glaive-coder-7b\n",
"# add the 'huggingface/' prefix to the model to set huggingface as the provider\n",
"# set api base to your deployed api endpoint from hugging face\n",
"response = litellm.completion(\n",
" model=\"huggingface/glaiveai/glaive-coder-7b\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" api_base=\"https://wjiegasee9bmqke2.us-east-1.aws.endpoints.huggingface.cloud\"\n",
")\n",
"print(response)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Lbmw8Gl_pHns",
"outputId": "ea8408bf-1cc3-4670-ecea-f12666d204a8"
},
"execution_count": 9,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"length\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"\\n\\nI am doing well, thank you for asking. How about you?\\nI am doing\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": -8.9481967812\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-74dc9d89-3916-47ce-9bea-b80e66660f77\",\n",
" \"created\": 1695871068.8413374,\n",
" \"model\": \"glaiveai/glaive-coder-7b\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 18,\n",
" \"total_tokens\": 24\n",
" }\n",
"}\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## HuggingFace Non TGI/Non Conversational Model - Deployed Inference Endpoints\n",
"* set `api_base` to your deployed api base\n",
"* Add the `huggingface/` prefix to your model so litellm knows it's a huggingface Deployed Inference Endpoint"
],
"metadata": {
"id": "WZNyq76syYyh"
}
},
{
"cell_type": "code",
"source": [
"import os\n",
"import litellm\n",
"\n",
"os.environ[\"HUGGINGFACE_API_KEY\"] = \"\"\n",
"# model: https://huggingface.co/roneneldan/TinyStories-3M\n",
"# add the 'huggingface/' prefix to the model to set huggingface as the provider\n",
"# set api base to your deployed api endpoint from hugging face\n",
"response = litellm.completion(\n",
" model=\"huggingface/roneneldan/TinyStories-3M\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" api_base=\"https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud\",\n",
" )\n",
"print(response)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "W8kMlXd6yRXu",
"outputId": "63e2cd7a-8759-4ee6-bac4-fe34ce8f0ca0"
},
"execution_count": 6,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"Hello, how are you? I have a surprise for you. I have a surprise for you.\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-6035abd6-7753-4a7d-ba0a-8193522e23cf\",\n",
" \"created\": 1695871015.0468287,\n",
" \"model\": \"roneneldan/TinyStories-3M\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 20,\n",
" \"total_tokens\": 26\n",
" }\n",
"}\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Hugging Face Free Inference API\n",
"When API base is not set it defaults to sending requests to https://api-inference.huggingface.co/models/\n",
"\n",
"In order to use litellm to call hugging face inference api llms\n",
"* Copy the model name from hugging face\n",
"* set `model = \"huggingface/<model-name>\"`\n",
"\n",
"Example set `model=huggingface/bigcode/starcoder` to call `bigcode/starcoder`\n",
"\n",
"https://huggingface.co/bigcode/starcoder"
],
"metadata": {
"id": "yp5UXRqtpu9f"
}
},
"source": [
"## Serverless Inference Providers\n",
"\n",
"Read more about Inference Providers here: https://huggingface.co/blog/inference-providers.\n",
"\n",
"In order to use litellm with Hugging Face Inference Providers, you need to set `model=huggingface/<provider>/<model-id>`.\n",
"\n",
"Example: `huggingface/together/deepseek-ai/DeepSeek-R1` to run DeepSeek-R1 (https://huggingface.co/deepseek-ai/DeepSeek-R1) through Together AI.\n"
]
},
{
"cell_type": "code",
"source": [
"import os\n",
"import litellm\n",
"\n",
"os.environ[\"HUGGINGFACE_API_KEY\"] = \"\"\n",
"\n",
"# Call https://huggingface.co/bigcode/starcoder\n",
"# add the 'huggingface/' prefix to the model to set huggingface as the provider\n",
"response = litellm.completion(\n",
" model=\"huggingface/bigcode/starcoder\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(response)\n",
"\n",
"\n",
"# Call https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf\n",
"response = litellm.completion(\n",
" model=\"huggingface/codellama/CodeLlama-34b-Instruct-hf\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
")\n",
"print(response)"
],
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -220,92 +47,34 @@
"id": "Pi5Oww8gpCUm",
"outputId": "659a67c7-f90d-4c06-b94e-2c4aa92d897a"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" I am fine, thank you. And you?')\\nprint(result)\\n\\n# 2\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-982e4cd0-9779-4108-9f7e-d6cbf9b71516\",\n",
" \"created\": 1695835548.2239568,\n",
" \"model\": \"bigcode/starcoder\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 6,\n",
" \"completion_tokens\": 17,\n",
" \"total_tokens\": 23\n",
" }\n",
"}\n",
"{\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \"Hello! I'm doing well, thank you for asking. It's nice to meet you\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-6622d64d-e9fc-4a46-9ca7-b2d011f6968c\",\n",
" \"created\": 1695835549.2932954,\n",
" \"model\": \"codellama/CodeLlama-34b-Instruct-hf\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 12,\n",
" \"completion_tokens\": 18,\n",
" \"total_tokens\": 30\n",
" }\n",
"}\n"
]
}
"outputs": [],
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"# You can create a HF token here: https://huggingface.co/settings/tokens\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"# Call DeepSeek-R1 model through Together AI\n",
"response = completion(\n",
" model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n",
" messages=[{\"content\": \"How many r's are in the word `strawberry`?\", \"role\": \"user\"}],\n",
")\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"source": [
"## HuggingFace - Deployed Inference Endpoints + Streaming\n",
"Set stream = True"
],
"metadata": {
"id": "EU0UubrKzTFe"
}
},
"source": [
"## Streaming\n"
]
},
{
"cell_type": "code",
"source": [
"import os\n",
"import litellm\n",
"\n",
"os.environ[\"HUGGINGFACE_API_KEY\"] = \"\"\n",
"\n",
"# Call https://huggingface.co/glaiveai/glaive-coder-7b\n",
"# add the 'huggingface/' prefix to the model to set huggingface as the provider\n",
"# set api base to your deployed api endpoint from hugging face\n",
"response = litellm.completion(\n",
" model=\"huggingface/aws-glaive-coder-7b-0998\",\n",
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
" api_base=\"https://wjiegasee9bmqke2.us-east-1.aws.endpoints.huggingface.cloud\",\n",
" stream=True\n",
")\n",
"print(response)\n",
"\n",
"for chunk in response:\n",
" print(chunk)"
],
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -313,446 +82,171 @@
"id": "y-QfIvA-uJKX",
"outputId": "b007bb98-00d0-44a4-8264-c8a2caed6768"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"<litellm.utils.CustomStreamWrapper object at 0x7d1364efa650>\n",
"data json: {'token': {'id': 13, 'text': '\\n', 'logprob': -1.4355469, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \"\\n\",\n",
" \"role\": \"assistant\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-b581bf7e-e20d-46fd-9ca0-b38870db3f3c\",\n",
" \"created\": 1695837652,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 13, 'text': '\\n', 'logprob': -1.9277344, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \"\\n\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-49c7b630-ec07-4390-ae22-bbb068ac66aa\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 29902, 'text': 'I', 'logprob': -1.4570312, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \"I\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-6b1635f3-810a-4976-b603-2c47a9525fff\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 626, 'text': ' am', 'logprob': -0.70703125, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" am\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-dcfad593-6c02-4f4c-abdb-3027ccda80e1\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 2599, 'text': ' doing', 'logprob': -1.0107422, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" doing\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-fcdd4076-7907-44f9-8ebf-a462b90e076c\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 1532, 'text': ' well', 'logprob': -0.43603516, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" well\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-c3e521f8-5cec-4a65-908a-f6678a635806\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 29892, 'text': ',', 'logprob': -0.08898926, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \",\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-c32355fb-94cc-43c8-9213-71ff387dc636\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 6452, 'text': ' thank', 'logprob': -0.19006348, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" thank\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-0ea8172e-adcf-4bcc-b919-df675d3def85\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 366, 'text': ' you', 'logprob': -0.0012788773, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" you\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-9c9fc627-fec9-454e-a630-6f8a2291b662\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 363, 'text': ' for', 'logprob': -0.026885986, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" for\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-fd32fd74-6cac-4ddb-83d1-205810ab897a\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 6721, 'text': ' asking', 'logprob': -0.035705566, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" asking\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-2ff7ab0a-d574-4e83-804f-8505be94712a\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 29889, 'text': '.', 'logprob': -0.07635498, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \".\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-488d6bba-23bf-4383-bced-3cc0fbad3b17\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 1128, 'text': ' How', 'logprob': -0.46557617, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" How\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-078c7ce3-e748-4fd7-bf11-e1389e23e0ef\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 1048, 'text': ' about', 'logprob': -0.068359375, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" about\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-d6b1e355-edf1-4486-8567-d4b5bbfd7d74\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 366, 'text': ' you', 'logprob': -0.0006146431, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" you\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-d2633371-0235-4a31-9621-4a9ec9b587a8\",\n",
" \"created\": 1695837653,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 29973, 'text': '?', 'logprob': -0.0001667738, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \"?\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-70ccf462-e915-465a-bb86-46f5a244451e\",\n",
" \"created\": 1695837654,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 13, 'text': '\\n', 'logprob': -0.03363037, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \"\\n\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-31d947e4-71df-4954-9d1a-8e68464da879\",\n",
" \"created\": 1695837654,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 29902, 'text': 'I', 'logprob': -0.17321777, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \"I\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-af84f782-a682-4660-86ba-da1ad71f5c93\",\n",
" \"created\": 1695837654,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 626, 'text': ' am', 'logprob': -0.38891602, 'special': False}, 'generated_text': None, 'details': None}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": null,\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" am\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-3d4e01aa-4d56-4b98-be8f-8f9a6a4e0856\",\n",
" \"created\": 1695837654,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n",
"data json: {'token': {'id': 2599, 'text': ' doing', 'logprob': -0.4243164, 'special': False}, 'generated_text': '\\n\\nI am doing well, thank you for asking. How about you?\\nI am doing', 'details': {'finish_reason': 'length', 'generated_tokens': 20, 'seed': None}}\n",
"{\n",
" \"object\": \"chat.completion.chunk\",\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"length\",\n",
" \"index\": 0,\n",
" \"delta\": {\n",
" \"content\": \" doing\"\n",
" }\n",
" }\n",
" ],\n",
" \"id\": \"chatcmpl-18583fad-c957-432e-9a62-5620620271a2\",\n",
" \"created\": 1695837654,\n",
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": null,\n",
" \"completion_tokens\": null,\n",
" \"total_tokens\": null\n",
" }\n",
"}\n"
]
}
"outputs": [],
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"response = completion(\n",
" model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n",
" messages=[\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"How many r's are in the word `strawberry`?\",\n",
" \n",
" }\n",
" ],\n",
" stream=True,\n",
")\n",
"\n",
"for chunk in response:\n",
" print(chunk)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## With images as input\n"
]
},
{
"cell_type": "code",
"source": [],
"metadata": {
"id": "CKXAnK55zQRl"
},
"execution_count": null,
"outputs": []
"metadata": {},
"outputs": [],
"source": [
"from litellm import completion\n",
"\n",
"# Set your Hugging Face Token\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": [\n",
" {\"type\": \"text\", \"text\": \"What's in this image?\"},\n",
" {\n",
" \"type\": \"image_url\",\n",
" \"image_url\": {\n",
" \"url\": \"https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png\",\n",
" },\n",
" },\n",
" ],\n",
" }\n",
"]\n",
"\n",
"response = completion(\n",
" model=\"huggingface/sambanova/meta-llama/Llama-3.3-70B-Instruct\",\n",
" messages=messages,\n",
")\n",
"print(response.choices[0])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Tools - Function Calling\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"\n",
"# Set your Hugging Face Token\n",
"os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n",
"\n",
"tools = [\n",
" {\n",
" \"type\": \"function\",\n",
" \"function\": {\n",
" \"name\": \"get_current_weather\",\n",
" \"description\": \"Get the current weather in a given location\",\n",
" \"parameters\": {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"location\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
" },\n",
" \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
" },\n",
" \"required\": [\"location\"],\n",
" },\n",
" },\n",
" }\n",
"]\n",
"messages = [{\"role\": \"user\", \"content\": \"What's the weather like in Boston today?\"}]\n",
"\n",
"response = completion(\n",
" model=\"huggingface/sambanova/meta-llama/Llama-3.1-8B-Instruct\", messages=messages, tools=tools, tool_choice=\"auto\"\n",
")\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Hugging Face Dedicated Inference Endpoints\n",
"\n",
"Steps to use\n",
"\n",
"- Create your own Hugging Face dedicated endpoint here: https://ui.endpoints.huggingface.co/\n",
"- Set `api_base` to your deployed api base\n",
"- set the model to `huggingface/tgi` so that litellm knows it's a huggingface Deployed Inference Endpoint.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import litellm\n",
"\n",
"\n",
"response = litellm.completion(\n",
" model=\"huggingface/tgi\",\n",
" messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}],\n",
" api_base=\"https://my-endpoint.endpoints.huggingface.cloud/v1/\",\n",
")\n",
"print(response)"
]
}
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.0"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

97
cookbook/LiteLLM_NovitaAI_Cookbook.ipynb vendored Normal file
View file

@ -0,0 +1,97 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "iFEmsVJI_2BR"
},
"source": [
"# LiteLLM NovitaAI Cookbook"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "cBlUhCEP_xj4"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "p-MQqWOT_1a7"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ['NOVITA_API_KEY'] = \"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Ze8JqMqWAARO"
},
"outputs": [],
"source": [
"from litellm import completion\n",
"response = completion(\n",
" model=\"novita/deepseek/deepseek-r1\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "-LnhELrnAM_J"
},
"outputs": [],
"source": [
"response = completion(\n",
" model=\"novita/deepseek/deepseek-r1\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "dJBOUYdwCEn1"
},
"outputs": [],
"source": [
"response = completion(\n",
" model=\"mistralai/mistral-7b-instruct\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

114
cookbook/LiteLLM_OpenRouter.ipynb vendored
View file

@ -1,27 +1,13 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# LiteLLM OpenRouter Cookbook"
],
"metadata": {
"id": "iFEmsVJI_2BR"
}
},
"source": [
"# LiteLLM OpenRouter Cookbook"
]
},
{
"cell_type": "code",
@ -36,27 +22,20 @@
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"id": "p-MQqWOT_1a7"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ['OPENROUTER_API_KEY'] = \"\""
],
"metadata": {
"id": "p-MQqWOT_1a7"
},
"execution_count": 14,
"outputs": []
]
},
{
"cell_type": "code",
"source": [
"from litellm import completion\n",
"response = completion(\n",
" model=\"openrouter/google/palm-2-chat-bison\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
],
"execution_count": 11,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -64,10 +43,8 @@
"id": "Ze8JqMqWAARO",
"outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98"
},
"execution_count": 11,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<OpenAIObject id=gen-W8FTMSIEorCp3vG5iYIgNMR4IeBv at 0x7c3dcef1f060> JSON: {\n",
@ -85,20 +62,23 @@
"}"
]
},
"execution_count": 11,
"metadata": {},
"execution_count": 11
"output_type": "execute_result"
}
],
"source": [
"from litellm import completion\n",
"response = completion(\n",
" model=\"openrouter/google/palm-2-chat-bison\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"source": [
"response = completion(\n",
" model=\"openrouter/anthropic/claude-2\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
],
"execution_count": 12,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -106,10 +86,8 @@
"id": "-LnhELrnAM_J",
"outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0"
},
"execution_count": 12,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<OpenAIObject id=gen-IiuV7ZNimDufVeutBHrl8ajPuzEh at 0x7c3dcea67560> JSON: {\n",
@ -128,20 +106,22 @@
"}"
]
},
"execution_count": 12,
"metadata": {},
"execution_count": 12
"output_type": "execute_result"
}
],
"source": [
"response = completion(\n",
" model=\"openrouter/anthropic/claude-2\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
},
{
"cell_type": "code",
"source": [
"response = completion(\n",
" model=\"openrouter/meta-llama/llama-2-70b-chat\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
],
"execution_count": 13,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -149,10 +129,8 @@
"id": "dJBOUYdwCEn1",
"outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0"
},
"execution_count": 13,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<OpenAIObject id=gen-PyMd3yyJ0aQsCgIY9R8XGZoAtPbl at 0x7c3dceefcae0> JSON: {\n",
@ -170,10 +148,32 @@
"}"
]
},
"execution_count": 13,
"metadata": {},
"execution_count": 13
"output_type": "execute_result"
}
],
"source": [
"response = completion(\n",
" model=\"openrouter/meta-llama/llama-2-70b-chat\",\n",
" messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n",
")\n",
"response"
]
}
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

2
cookbook/LiteLLM_PromptLayer.ipynb vendored
View file

@ -39,7 +39,7 @@
"import os\n",
"os.environ['OPENAI_API_KEY'] = \"\"\n",
"os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
"os.environ['PROMPTLAYER_API_KEY'] = \"pl_4ea2bb00a4dca1b8a70cebf2e9e11564\"\n",
"os.environ['PROMPTLAYER_API_KEY'] = \"test-promptlayer-key-123\"\n",
"\n",
"# Set Promptlayer as a success callback\n",
"litellm.success_callback =['promptlayer']\n",

315
cookbook/LiteLLM_batch_completion.ipynb vendored
View file

@ -1,166 +1,163 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "MbLbs1tbISk-"
},
"source": [
"# LiteLLM Batch Completions Example\n",
"\n",
"* This tutorial walks through using `batch_completion`\n",
"* Docs: https://docs.litellm.ai/docs/completion/batching"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# LiteLLM Batch Completions Example\n",
"\n",
"* This tutorial walks through using `batch_completion`\n",
"* Docs: https://docs.litellm.ai/docs/completion/batching"
],
"metadata": {
"id": "MbLbs1tbISk-"
}
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Ty6-ko_aDlPF"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "KGhNJRUCIh1j"
},
"source": [
"## Import Batch Completion"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"id": "LOtI43snDrSK"
},
"outputs": [],
"source": [
"import os\n",
"from litellm import batch_completion\n",
"\n",
"# set your API_KEY\n",
"os.environ['ANTHROPIC_API_KEY'] = \"\""
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Xhv92NBaIpaw"
},
"source": [
"## Calling `litellm.batch_completion`\n",
"\n",
"In the batch_completion method, you provide a list of messages where each sub-list of messages is passed to litellm.completion(), allowing you to process multiple prompts efficiently in a single API call."
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "yY7GIRLsDywu",
"outputId": "009ea67f-95d5-462b-947f-b0d21e60c5bb"
},
"outputs": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Ty6-ko_aDlPF"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
{
"cell_type": "markdown",
"source": [
"## Import Batch Completion"
],
"metadata": {
"id": "KGhNJRUCIh1j"
}
},
{
"cell_type": "code",
"source": [
"import litellm\n",
"import os\n",
"from litellm import batch_completion\n",
"\n",
"# set your API_KEY\n",
"os.environ['ANTHROPIC_API_KEY'] = \"\""
],
"metadata": {
"id": "LOtI43snDrSK"
},
"execution_count": 7,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Calling `litellm.batch_completion`\n",
"\n",
"In the batch_completion method, you provide a list of messages where each sub-list of messages is passed to litellm.completion(), allowing you to process multiple prompts efficiently in a single API call."
],
"metadata": {
"id": "Xhv92NBaIpaw"
}
},
{
"cell_type": "code",
"source": [
"import litellm\n",
"import os\n",
"from litellm import batch_completion\n",
"\n",
"os.environ['ANTHROPIC_API_KEY'] = \"\"\n",
"\n",
"\n",
"responses = batch_completion(\n",
" model=\"claude-2\",\n",
" messages = [\n",
" [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"good morning? \"\n",
" }\n",
" ],\n",
" [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"what's the time? \"\n",
" }\n",
" ]\n",
" ]\n",
")\n",
"responses"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "yY7GIRLsDywu",
"outputId": "009ea67f-95d5-462b-947f-b0d21e60c5bb"
},
"execution_count": 11,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"[<ModelResponse at 0x7a164eed4450> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" Good morning!\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1694030351.309254,\n",
" \"model\": \"claude-2\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 11,\n",
" \"completion_tokens\": 3,\n",
" \"total_tokens\": 14\n",
" }\n",
" },\n",
" <ModelResponse at 0x7a164eed5800> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" I'm an AI assistant created by Anthropic. I don't actually have a concept of the current time.\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1694030352.1215081,\n",
" \"model\": \"claude-2\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 13,\n",
" \"completion_tokens\": 22,\n",
" \"total_tokens\": 35\n",
" }\n",
" }]"
]
},
"metadata": {},
"execution_count": 11
}
"data": {
"text/plain": [
"[<ModelResponse at 0x7a164eed4450> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" Good morning!\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1694030351.309254,\n",
" \"model\": \"claude-2\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 11,\n",
" \"completion_tokens\": 3,\n",
" \"total_tokens\": 14\n",
" }\n",
" },\n",
" <ModelResponse at 0x7a164eed5800> JSON: {\n",
" \"choices\": [\n",
" {\n",
" \"finish_reason\": \"stop\",\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"content\": \" I'm an AI assistant created by Anthropic. I don't actually have a concept of the current time.\",\n",
" \"role\": \"assistant\",\n",
" \"logprobs\": null\n",
" }\n",
" }\n",
" ],\n",
" \"created\": 1694030352.1215081,\n",
" \"model\": \"claude-2\",\n",
" \"usage\": {\n",
" \"prompt_tokens\": 13,\n",
" \"completion_tokens\": 22,\n",
" \"total_tokens\": 35\n",
" }\n",
" }]"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
]
],
"source": [
"import os\n",
"\n",
"os.environ['ANTHROPIC_API_KEY'] = \"\"\n",
"\n",
"\n",
"responses = batch_completion(\n",
" model=\"claude-2\",\n",
" messages = [\n",
" [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"good morning? \"\n",
" }\n",
" ],\n",
" [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"what's the time? \"\n",
" }\n",
" ]\n",
" ]\n",
")\n",
"responses"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

565
cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb vendored Normal file
View file

@ -0,0 +1,565 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "kccfk0mHZ4Ad"
},
"source": [
"# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n",
"\n",
"Covers:\n",
"\n",
"* /chat/completion\n",
"* /embedding\n",
"\n",
"\n",
"These are **selected examples**. LiteLLM Proxy is **OpenAI-Compatible**, it works with any project that calls OpenAI. Just change the `base_url`, `api_key` and `model`.\n",
"\n",
"For more examples, [go here](https://docs.litellm.ai/docs/proxy/user_keys)\n",
"\n",
"To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
"\n",
"To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "nmSClzCPaGH6"
},
"source": [
"## /chat/completion\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "_vqcjwOVaKpO"
},
"source": [
"### OpenAI Python SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "x1e_Ok3KZzeP"
},
"outputs": [],
"source": [
"import openai\n",
"client = openai.OpenAI(\n",
" api_key=\"anything\",\n",
" base_url=\"http://0.0.0.0:4000\"\n",
")\n",
"\n",
"# request sent to model set on litellm proxy, `litellm --model`\n",
"response = client.chat.completions.create(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"this is a test request, write a short poem\"\n",
" }\n",
" ],\n",
" extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params\n",
" \"metadata\": { # 👈 use for logging additional params (e.g. to langfuse)\n",
" \"generation_name\": \"ishaan-generation-openai-client\",\n",
" \"generation_id\": \"openai-client-gen-id22\",\n",
" \"trace_id\": \"openai-client-trace-id22\",\n",
" \"trace_user_id\": \"openai-client-user-id2\"\n",
" }\n",
" }\n",
")\n",
"\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "AqkyKk9Scxgj"
},
"source": [
"## Function Calling"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "wDg10VqLczE1"
},
"outputs": [],
"source": [
"from openai import OpenAI\n",
"client = OpenAI(\n",
" api_key=\"sk-1234\", # [OPTIONAL] set if you set one on proxy, else set \"\"\n",
" base_url=\"http://0.0.0.0:4000\",\n",
")\n",
"\n",
"tools = [\n",
" {\n",
" \"type\": \"function\",\n",
" \"function\": {\n",
" \"name\": \"get_current_weather\",\n",
" \"description\": \"Get the current weather in a given location\",\n",
" \"parameters\": {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"location\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
" },\n",
" \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n",
" },\n",
" \"required\": [\"location\"],\n",
" },\n",
" }\n",
" }\n",
"]\n",
"messages = [{\"role\": \"user\", \"content\": \"What's the weather like in Boston today?\"}]\n",
"completion = client.chat.completions.create(\n",
" model=\"gpt-4o\", # use 'model_name' from config.yaml\n",
" messages=messages,\n",
" tools=tools,\n",
" tool_choice=\"auto\"\n",
")\n",
"\n",
"print(completion)\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "YYoxLloSaNWW"
},
"source": [
"### Azure OpenAI Python SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "yA1XcgowaSRy"
},
"outputs": [],
"source": [
"import openai\n",
"client = openai.AzureOpenAI(\n",
" api_key=\"anything\",\n",
" base_url=\"http://0.0.0.0:4000\"\n",
")\n",
"\n",
"# request sent to model set on litellm proxy, `litellm --model`\n",
"response = client.chat.completions.create(\n",
" model=\"gpt-3.5-turbo\",\n",
" messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"this is a test request, write a short poem\"\n",
" }\n",
" ],\n",
" extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params\n",
" \"metadata\": { # 👈 use for logging additional params (e.g. to langfuse)\n",
" \"generation_name\": \"ishaan-generation-openai-client\",\n",
" \"generation_id\": \"openai-client-gen-id22\",\n",
" \"trace_id\": \"openai-client-trace-id22\",\n",
" \"trace_user_id\": \"openai-client-user-id2\"\n",
" }\n",
" }\n",
")\n",
"\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "yl9qhDvnaTpL"
},
"source": [
"### Langchain Python"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "5MUZgSquaW5t"
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.prompts.chat import (\n",
" ChatPromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
" SystemMessagePromptTemplate,\n",
")\n",
"from langchain.schema import HumanMessage, SystemMessage\n",
"import os\n",
"\n",
"os.environ[\"OPENAI_API_KEY\"] = \"anything\"\n",
"\n",
"chat = ChatOpenAI(\n",
" openai_api_base=\"http://0.0.0.0:4000\",\n",
" model = \"gpt-3.5-turbo\",\n",
" temperature=0.1,\n",
" extra_body={\n",
" \"metadata\": {\n",
" \"generation_name\": \"ishaan-generation-langchain-client\",\n",
" \"generation_id\": \"langchain-client-gen-id22\",\n",
" \"trace_id\": \"langchain-client-trace-id22\",\n",
" \"trace_user_id\": \"langchain-client-user-id2\"\n",
" }\n",
" }\n",
")\n",
"\n",
"messages = [\n",
" SystemMessage(\n",
" content=\"You are a helpful assistant that im using to make a test request to.\"\n",
" ),\n",
" HumanMessage(\n",
" content=\"test from litellm. tell me why it's amazing in 1 sentence\"\n",
" ),\n",
"]\n",
"response = chat(messages)\n",
"\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "B9eMgnULbRaz"
},
"source": [
"### Curl"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "VWCCk5PFcmhS"
},
"source": [
"\n",
"\n",
"```\n",
"curl -X POST 'http://0.0.0.0:4000/chat/completions' \\\n",
" -H 'Content-Type: application/json' \\\n",
" -d '{\n",
" \"model\": \"gpt-3.5-turbo\",\n",
" \"messages\": [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"what llm are you\"\n",
" }\n",
" ],\n",
" \"metadata\": {\n",
" \"generation_name\": \"ishaan-test-generation\",\n",
" \"generation_id\": \"gen-id22\",\n",
" \"trace_id\": \"trace-id22\",\n",
" \"trace_user_id\": \"user-id2\"\n",
" }\n",
"}'\n",
"```\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "drBAm2e1b6xe"
},
"source": [
"### LlamaIndex"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "d0bZcv8fb9mL"
},
"outputs": [],
"source": [
"import os, dotenv\n",
"\n",
"from llama_index.llms import AzureOpenAI\n",
"from llama_index.embeddings import AzureOpenAIEmbedding\n",
"from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext\n",
"\n",
"llm = AzureOpenAI(\n",
" engine=\"azure-gpt-3.5\", # model_name on litellm proxy\n",
" temperature=0.0,\n",
" azure_endpoint=\"http://0.0.0.0:4000\", # litellm proxy endpoint\n",
" api_key=\"sk-1234\", # litellm proxy API Key\n",
" api_version=\"2023-07-01-preview\",\n",
")\n",
"\n",
"embed_model = AzureOpenAIEmbedding(\n",
" deployment_name=\"azure-embedding-model\",\n",
" azure_endpoint=\"http://0.0.0.0:4000\",\n",
" api_key=\"sk-1234\",\n",
" api_version=\"2023-07-01-preview\",\n",
")\n",
"\n",
"\n",
"documents = SimpleDirectoryReader(\"llama_index_data\").load_data()\n",
"service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)\n",
"index = VectorStoreIndex.from_documents(documents, service_context=service_context)\n",
"\n",
"query_engine = index.as_query_engine()\n",
"response = query_engine.query(\"What did the author do growing up?\")\n",
"print(response)\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "xypvNdHnb-Yy"
},
"source": [
"### Langchain JS"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "R55mK2vCcBN2"
},
"outputs": [],
"source": [
"import { ChatOpenAI } from \"@langchain/openai\";\n",
"\n",
"\n",
"const model = new ChatOpenAI({\n",
" modelName: \"gpt-4\",\n",
" openAIApiKey: \"sk-1234\",\n",
" modelKwargs: {\"metadata\": \"hello world\"} // 👈 PASS Additional params here\n",
"}, {\n",
" basePath: \"http://0.0.0.0:4000\",\n",
"});\n",
"\n",
"const message = await model.invoke(\"Hi there!\");\n",
"\n",
"console.log(message);\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "nC4bLifCcCiW"
},
"source": [
"### OpenAI JS"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "MICH8kIMcFpg"
},
"outputs": [],
"source": [
"const { OpenAI } = require('openai');\n",
"\n",
"const openai = new OpenAI({\n",
" apiKey: \"sk-1234\", // This is the default and can be omitted\n",
" baseURL: \"http://0.0.0.0:4000\"\n",
"});\n",
"\n",
"async function main() {\n",
" const chatCompletion = await openai.chat.completions.create({\n",
" messages: [{ role: 'user', content: 'Say this is a test' }],\n",
" model: 'gpt-3.5-turbo',\n",
" }, {\"metadata\": {\n",
" \"generation_name\": \"ishaan-generation-openaijs-client\",\n",
" \"generation_id\": \"openaijs-client-gen-id22\",\n",
" \"trace_id\": \"openaijs-client-trace-id22\",\n",
" \"trace_user_id\": \"openaijs-client-user-id2\"\n",
" }});\n",
"}\n",
"\n",
"main();\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "D1Q07pEAcGTb"
},
"source": [
"### Anthropic SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "qBjFcAvgcI3t"
},
"outputs": [],
"source": [
"import os\n",
"\n",
"from anthropic import Anthropic\n",
"\n",
"client = Anthropic(\n",
" base_url=\"http://localhost:4000\", # proxy endpoint\n",
" api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n",
")\n",
"\n",
"message = client.messages.create(\n",
" max_tokens=1024,\n",
" messages=[\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"Hello, Claude\",\n",
" }\n",
" ],\n",
" model=\"claude-3-opus-20240229\",\n",
")\n",
"print(message.content)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "dFAR4AJGcONI"
},
"source": [
"## /embeddings"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "lgNoM281cRzR"
},
"source": [
"### OpenAI Python SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "NY3DJhPfcQhA"
},
"outputs": [],
"source": [
"import openai\n",
"from openai import OpenAI\n",
"\n",
"# set base_url to your proxy server\n",
"# set api_key to send to proxy server\n",
"client = OpenAI(api_key=\"<proxy-api-key>\", base_url=\"http://0.0.0.0:4000\")\n",
"\n",
"response = client.embeddings.create(\n",
" input=[\"hello from litellm\"],\n",
" model=\"text-embedding-ada-002\"\n",
")\n",
"\n",
"print(response)\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "hmbg-DW6cUZs"
},
"source": [
"### Langchain Embeddings"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "lX2S8Nl1cWVP"
},
"outputs": [],
"source": [
"from langchain.embeddings import OpenAIEmbeddings\n",
"\n",
"embeddings = OpenAIEmbeddings(model=\"sagemaker-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n",
"\n",
"\n",
"text = \"This is a test document.\"\n",
"\n",
"query_result = embeddings.embed_query(text)\n",
"\n",
"print(f\"SAGEMAKER EMBEDDINGS\")\n",
"print(query_result[:5])\n",
"\n",
"embeddings = OpenAIEmbeddings(model=\"bedrock-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n",
"\n",
"text = \"This is a test document.\"\n",
"\n",
"query_result = embeddings.embed_query(text)\n",
"\n",
"print(f\"BEDROCK EMBEDDINGS\")\n",
"print(query_result[:5])\n",
"\n",
"embeddings = OpenAIEmbeddings(model=\"bedrock-titan-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n",
"\n",
"text = \"This is a test document.\"\n",
"\n",
"query_result = embeddings.embed_query(text)\n",
"\n",
"print(f\"TITAN EMBEDDINGS\")\n",
"print(query_result[:5])"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "oqGbWBCQcYfd"
},
"source": [
"### Curl Request"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7rkIMV9LcdwQ"
},
"source": [
"\n",
"\n",
"```curl\n",
"curl -X POST 'http://0.0.0.0:4000/embeddings' \\\n",
" -H 'Content-Type: application/json' \\\n",
" -d ' {\n",
" \"model\": \"text-embedding-ada-002\",\n",
" \"input\": [\"write a litellm poem\"]\n",
" }'\n",
"```\n",
"\n"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

205
cookbook/Proxy_Batch_Users.ipynb vendored Normal file
View file

@ -0,0 +1,205 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "680oRk1af-xJ"
},
"source": [
"# Environment Setup"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "X7TgJFn8f88p"
},
"outputs": [],
"source": [
"import csv\n",
"from typing import Optional\n",
"import httpx\n",
"import json\n",
"import asyncio\n",
"\n",
"proxy_base_url = \"http://0.0.0.0:4000\" # 👈 SET TO PROXY URL\n",
"master_key = \"sk-1234\" # 👈 SET TO PROXY MASTER KEY"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "rauw8EOhgBz5"
},
"outputs": [],
"source": [
"## GLOBAL HTTP CLIENT ## - faster http calls\n",
"class HTTPHandler:\n",
" def __init__(self, concurrent_limit=1000):\n",
" # Create a client with a connection pool\n",
" self.client = httpx.AsyncClient(\n",
" limits=httpx.Limits(\n",
" max_connections=concurrent_limit,\n",
" max_keepalive_connections=concurrent_limit,\n",
" )\n",
" )\n",
"\n",
" async def close(self):\n",
" # Close the client when you're done with it\n",
" await self.client.aclose()\n",
"\n",
" async def get(\n",
" self, url: str, params: Optional[dict] = None, headers: Optional[dict] = None\n",
" ):\n",
" response = await self.client.get(url, params=params, headers=headers)\n",
" return response\n",
"\n",
" async def post(\n",
" self,\n",
" url: str,\n",
" data: Optional[dict] = None,\n",
" params: Optional[dict] = None,\n",
" headers: Optional[dict] = None,\n",
" ):\n",
" try:\n",
" response = await self.client.post(\n",
" url, data=data, params=params, headers=headers\n",
" )\n",
" return response\n",
" except Exception as e:\n",
" raise e\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7LXN8zaLgOie"
},
"source": [
"# Import Sheet\n",
"\n",
"\n",
"Format: | ID | Name | Max Budget |"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "oiED0usegPGf"
},
"outputs": [],
"source": [
"async def import_sheet():\n",
" tasks = []\n",
" http_client = HTTPHandler()\n",
" with open('my-batch-sheet.csv', 'r') as file:\n",
" csv_reader = csv.DictReader(file)\n",
" for row in csv_reader:\n",
" task = create_user(client=http_client, user_id=row['ID'], max_budget=row['Max Budget'], user_name=row['Name'])\n",
" tasks.append(task)\n",
" # print(f\"ID: {row['ID']}, Name: {row['Name']}, Max Budget: {row['Max Budget']}\")\n",
"\n",
" keys = await asyncio.gather(*tasks)\n",
"\n",
" with open('my-batch-sheet_new.csv', 'w', newline='') as new_file:\n",
" fieldnames = ['ID', 'Name', 'Max Budget', 'keys']\n",
" csv_writer = csv.DictWriter(new_file, fieldnames=fieldnames)\n",
" csv_writer.writeheader()\n",
"\n",
" with open('my-batch-sheet.csv', 'r') as file:\n",
" csv_reader = csv.DictReader(file)\n",
" for i, row in enumerate(csv_reader):\n",
" row['keys'] = keys[i] # Add the 'keys' value from the corresponding task result\n",
" csv_writer.writerow(row)\n",
"\n",
" await http_client.close()\n",
"\n",
"asyncio.run(import_sheet())"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "E7M0Li_UgJeZ"
},
"source": [
"# Create Users + Keys\n",
"\n",
"- Creates a user\n",
"- Creates a key with max budget"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "NZudRFujf7j-"
},
"outputs": [],
"source": [
"\n",
"async def create_key_with_alias(client: HTTPHandler, user_id: str, max_budget: float):\n",
" global proxy_base_url\n",
" if not proxy_base_url.endswith(\"/\"):\n",
" proxy_base_url += \"/\"\n",
" url = proxy_base_url + \"key/generate\"\n",
"\n",
" # call /key/generate\n",
" print(\"CALLING /KEY/GENERATE\")\n",
" response = await client.post(\n",
" url=url,\n",
" headers={\"Authorization\": f\"Bearer {master_key}\"},\n",
" data=json.dumps({\n",
" \"user_id\": user_id,\n",
" \"key_alias\": f\"{user_id}-key\",\n",
" \"max_budget\": max_budget # 👈 KEY CHANGE: SETS MAX BUDGET PER KEY\n",
" })\n",
" )\n",
" print(f\"response: {response.text}\")\n",
" return response.json()[\"key\"]\n",
"\n",
"async def create_user(client: HTTPHandler, user_id: str, max_budget: float, user_name: str):\n",
" \"\"\"\n",
" - call /user/new\n",
" - create key for user\n",
" \"\"\"\n",
" global proxy_base_url\n",
" if not proxy_base_url.endswith(\"/\"):\n",
" proxy_base_url += \"/\"\n",
" url = proxy_base_url + \"user/new\"\n",
"\n",
" # call /user/new\n",
" await client.post(\n",
" url=url,\n",
" headers={\"Authorization\": f\"Bearer {master_key}\"},\n",
" data=json.dumps({\n",
" \"user_id\": user_id,\n",
" \"user_alias\": user_name,\n",
" \"auto_create_key\": False,\n",
" # \"max_budget\": max_budget # 👈 [OPTIONAL] Sets max budget per user (if you want to set a max budget across keys)\n",
" })\n",
" )\n",
"\n",
" # create key for user\n",
" return await create_key_with_alias(client=client, user_id=user_id, max_budget=max_budget)\n"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

2007
cookbook/TogetherAI_liteLLM.ipynb vendored
View file

File diff suppressed because it is too large Load diff

310
cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb vendored
View file

@ -1,159 +1,157 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "eKXncoQbU_2j"
},
"source": [
"# Using Nemo-Guardrails with LiteLLM Server\n",
"\n",
"[Call Bedrock, TogetherAI, Huggingface, etc. on the server](https://docs.litellm.ai/docs/providers)"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# Using Nemo-Guardrails with LiteLLM Server\n",
"\n",
"[Call Bedrock, TogetherAI, Huggingface, etc. on the server](https://docs.litellm.ai/docs/providers)"
],
"metadata": {
"id": "eKXncoQbU_2j"
}
},
{
"cell_type": "markdown",
"source": [
"## Using with Bedrock\n",
"\n",
"`docker run -e PORT=8000 -e AWS_ACCESS_KEY_ID=<your-aws-access-key> -e AWS_SECRET_ACCESS_KEY=<your-aws-secret-key> -p 8000:8000 ghcr.io/berriai/litellm:latest`"
],
"metadata": {
"id": "ZciYaLwvuFbu"
}
},
{
"cell_type": "code",
"source": [
"pip install nemoguardrails langchain"
],
"metadata": {
"id": "vOUwGSJ2Vsy3"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "xXEJNxe7U0IN"
},
"outputs": [],
"source": [
"import openai\n",
"from langchain.chat_models import ChatOpenAI\n",
"\n",
"llm = ChatOpenAI(model_name=\"anthropic.claude-v2\", openai_api_base=\"http://0.0.0.0:8000\", openai_api_key=\"my-fake-key\")\n",
"\n",
"from nemoguardrails import LLMRails, RailsConfig\n",
"\n",
"config = RailsConfig.from_path(\"./config.yml\")\n",
"app = LLMRails(config, llm=llm)\n",
"\n",
"new_message = app.generate(messages=[{\n",
" \"role\": \"user\",\n",
" \"content\": \"Hello! What can you do for me?\"\n",
"}])"
]
},
{
"cell_type": "markdown",
"source": [
"## Using with TogetherAI\n",
"\n",
"1. You can either set this in the server environment:\n",
"`docker run -e PORT=8000 -e TOGETHERAI_API_KEY=<your-together-ai-api-key> -p 8000:8000 ghcr.io/berriai/litellm:latest`\n",
"\n",
"2. **Or** Pass this in as the api key `(...openai_api_key=\"<your-together-ai-api-key>\")`"
],
"metadata": {
"id": "vz5n00qyuKjp"
}
},
{
"cell_type": "code",
"source": [
"import openai\n",
"from langchain.chat_models import ChatOpenAI\n",
"\n",
"llm = ChatOpenAI(model_name=\"together_ai/togethercomputer/CodeLlama-13b-Instruct\", openai_api_base=\"http://0.0.0.0:8000\", openai_api_key=\"my-together-ai-api-key\")\n",
"\n",
"from nemoguardrails import LLMRails, RailsConfig\n",
"\n",
"config = RailsConfig.from_path(\"./config.yml\")\n",
"app = LLMRails(config, llm=llm)\n",
"\n",
"new_message = app.generate(messages=[{\n",
" \"role\": \"user\",\n",
" \"content\": \"Hello! What can you do for me?\"\n",
"}])"
],
"metadata": {
"id": "XK1sk-McuhpE"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"### CONFIG.YML\n",
"\n",
"save this example `config.yml` in your current directory"
],
"metadata": {
"id": "8A1KWKnzuxAS"
}
},
{
"cell_type": "code",
"source": [
"# instructions:\n",
"# - type: general\n",
"# content: |\n",
"# Below is a conversation between a bot and a user about the recent job reports.\n",
"# The bot is factual and concise. If the bot does not know the answer to a\n",
"# question, it truthfully says it does not know.\n",
"\n",
"# sample_conversation: |\n",
"# user \"Hello there!\"\n",
"# express greeting\n",
"# bot express greeting\n",
"# \"Hello! How can I assist you today?\"\n",
"# user \"What can you do for me?\"\n",
"# ask about capabilities\n",
"# bot respond about capabilities\n",
"# \"I am an AI assistant that helps answer mathematical questions. My core mathematical skills are powered by wolfram alpha.\"\n",
"# user \"What's 2+2?\"\n",
"# ask math question\n",
"# bot responds to math question\n",
"# \"2+2 is equal to 4.\"\n",
"\n",
"# models:\n",
"# - type: main\n",
"# engine: openai\n",
"# model: claude-instant-1"
],
"metadata": {
"id": "NKN1GmSvu0Cx"
},
"execution_count": null,
"outputs": []
}
]
{
"cell_type": "markdown",
"metadata": {
"id": "ZciYaLwvuFbu"
},
"source": [
"## Using with Bedrock\n",
"\n",
"`docker run -e PORT=8000 -e AWS_ACCESS_KEY_ID=<your-aws-access-key> -e AWS_SECRET_ACCESS_KEY=<your-aws-secret-key> -p 8000:8000 ghcr.io/berriai/litellm:latest`"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "vOUwGSJ2Vsy3"
},
"outputs": [],
"source": [
"pip install nemoguardrails langchain"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "xXEJNxe7U0IN"
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"\n",
"llm = ChatOpenAI(model_name=\"anthropic.claude-v2\", openai_api_base=\"http://0.0.0.0:8000\", openai_api_key=\"my-fake-key\")\n",
"\n",
"from nemoguardrails import LLMRails, RailsConfig\n",
"\n",
"config = RailsConfig.from_path(\"./config.yml\")\n",
"app = LLMRails(config, llm=llm)\n",
"\n",
"new_message = app.generate(messages=[{\n",
" \"role\": \"user\",\n",
" \"content\": \"Hello! What can you do for me?\"\n",
"}])"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "vz5n00qyuKjp"
},
"source": [
"## Using with TogetherAI\n",
"\n",
"1. You can either set this in the server environment:\n",
"`docker run -e PORT=8000 -e TOGETHERAI_API_KEY=<your-together-ai-api-key> -p 8000:8000 ghcr.io/berriai/litellm:latest`\n",
"\n",
"2. **Or** Pass this in as the api key `(...openai_api_key=\"<your-together-ai-api-key>\")`"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "XK1sk-McuhpE"
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"\n",
"llm = ChatOpenAI(model_name=\"together_ai/togethercomputer/CodeLlama-13b-Instruct\", openai_api_base=\"http://0.0.0.0:8000\", openai_api_key=\"my-together-ai-api-key\")\n",
"\n",
"from nemoguardrails import LLMRails, RailsConfig\n",
"\n",
"config = RailsConfig.from_path(\"./config.yml\")\n",
"app = LLMRails(config, llm=llm)\n",
"\n",
"new_message = app.generate(messages=[{\n",
" \"role\": \"user\",\n",
" \"content\": \"Hello! What can you do for me?\"\n",
"}])"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "8A1KWKnzuxAS"
},
"source": [
"### CONFIG.YML\n",
"\n",
"save this example `config.yml` in your current directory"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "NKN1GmSvu0Cx"
},
"outputs": [],
"source": [
"# instructions:\n",
"# - type: general\n",
"# content: |\n",
"# Below is a conversation between a bot and a user about the recent job reports.\n",
"# The bot is factual and concise. If the bot does not know the answer to a\n",
"# question, it truthfully says it does not know.\n",
"\n",
"# sample_conversation: |\n",
"# user \"Hello there!\"\n",
"# express greeting\n",
"# bot express greeting\n",
"# \"Hello! How can I assist you today?\"\n",
"# user \"What can you do for me?\"\n",
"# ask about capabilities\n",
"# bot respond about capabilities\n",
"# \"I am an AI assistant that helps answer mathematical questions. My core mathematical skills are powered by wolfram alpha.\"\n",
"# user \"What's 2+2?\"\n",
"# ask math question\n",
"# bot responds to math question\n",
"# \"2+2 is equal to 4.\"\n",
"\n",
"# models:\n",
"# - type: main\n",
"# engine: openai\n",
"# model: claude-instant-1"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

295
cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md Normal file
View file

@ -0,0 +1,295 @@
# Claude Code with LiteLLM Quickstart
This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code.
> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
## Video Walkthrough
Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56
## Prerequisites
- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
- API keys for your chosen providers
## Installation
First, install LiteLLM with proxy support:
```bash
pip install 'litellm[proxy]'
```
## Step 1: Setup config.yaml
Create a secure configuration using environment variables:
```yaml
model_list:
# Claude models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
Set your environment variables:
```bash
export ANTHROPIC_API_KEY="your-anthropic-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
## Step 2: Start Proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
## Step 3: Verify Setup
Test that your proxy is working correctly:
```bash
curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "claude-3-5-sonnet-20241022",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
## Step 4: Configure Claude Code
### Method 1: Unified Endpoint (Recommended)
Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI.
### Method 2: Provider-specific Pass-through Endpoint
Alternatively, use the Anthropic pass-through endpoint:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
## Step 5: Use Claude Code
### Choosing Your Model
You have two options for specifying which model Claude Code uses:
#### Option 1: Command Line / Session Model Selection
Specify the model directly when starting Claude Code or during a session:
```bash
# Specify model at startup
claude --model claude-3-5-sonnet-20241022
# Or change model during a session
/model claude-3-5-haiku-20241022
```
This method uses the exact model you specify.
#### Option 2: Environment Variables
Configure default models using environment variables:
```bash
# Tell Claude Code which models to use by default
export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
export ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-3-5-20240229
claude # Will use the models specified above
```
**Note:** Claude Code may cache the model from a previous session. If environment variables don't take effect, use Option 1 to explicitly set the model.
**Important:** The `model_name` in your LiteLLM config must match what Claude Code requests (either from env vars or command line).
### Using 1M Context Window
Claude Code supports extended context (1 million tokens) using the `[1m]` suffix with Claude 4+ models:
```bash
# Use Sonnet 4.5 with 1M context (requires quotes for shell)
claude --model 'claude-sonnet-4-5-20250929[1m]'
# Inside a Claude Code session (no quotes needed)
/model claude-sonnet-4-5-20250929[1m]
```
**Important:** When using `--model` with `[1m]` in the shell, you must use quotes to prevent the shell from interpreting the brackets.
Alternatively, set as default with environment variables:
```bash
export ANTHROPIC_DEFAULT_SONNET_MODEL='claude-sonnet-4-5-20250929[1m]'
claude
```
**How it works:**
- Claude Code strips the `[1m]` suffix before sending to LiteLLM
- Claude Code automatically adds the header `anthropic-beta: context-1m-2025-08-07`
- Your LiteLLM config should **NOT** include `[1m]` in model names
**Verify 1M context is active:**
```bash
/context
# Should show: 21k/1000k tokens (2%)
```
**Pricing:** Models using 1M context have different pricing. Input tokens above 200k are charged at a higher rate.
## Troubleshooting
Common issues and solutions:
**Claude Code not connecting:**
- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
- Check that `ANTHROPIC_BASE_URL` is set correctly
- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Authentication errors:**
- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
- Check that your API keys are valid and have sufficient credits
- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Model not found:**
- Check what model Claude Code is requesting in LiteLLM logs
- Ensure your `config.yaml` has a matching `model_name` entry
- If using environment variables, verify they're set: `echo $ANTHROPIC_DEFAULT_SONNET_MODEL`
**1M context not working (showing 200k instead of 1000k):**
- Verify you're using the `[1m]` suffix: `/model your-model-name[1m]`
- Check LiteLLM logs for the header `context-1m-2025-08-07` in the request
- Ensure your model supports 1M context (only certain Claude models do)
- Your LiteLLM config should **NOT** include `[1m]` in the `model_name`
## Using Multiple Models and Providers
You can configure LiteLLM to route to any supported provider. Here's an example with multiple providers:
```yaml
model_list:
# OpenAI models
- model_name: codex-mini
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: o3-pro
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
# Anthropic models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
**Note:** The `model_name` can be anything you choose. Claude Code will request whatever model you specify (via env vars or command line), and LiteLLM will route to the `model` configured in `litellm_params`.
Switch between models seamlessly:
```bash
# Use environment variables to set defaults
export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
# Or specify directly
claude --model claude-3-5-sonnet-20241022 # Complex reasoning
claude --model claude-3-5-haiku-20241022 # Fast responses
claude --model claude-bedrock # Bedrock deployment
```
## Default Models Used by Claude Code
If you **don't** set environment variables, Claude Code uses these default model names:
| Purpose | Default Model Name (v2.1.14) |
|---------|------------------------------|
| Main model | `claude-sonnet-4-5-20250929` |
| Light tasks (subagents, summaries) | `claude-haiku-4-5-20251001` |
| Planning mode | `claude-opus-4-5-20251101` |
Your LiteLLM config should include these model names if you want Claude Code to work without setting environment variables:
```yaml
model_list:
- model_name: claude-sonnet-4-5-20250929
litellm_params:
# Can be any provider - Anthropic, Bedrock, Vertex AI, etc.
model: anthropic/claude-sonnet-4-5-20250929
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-haiku-4-5-20251001
litellm_params:
model: anthropic/claude-haiku-4-5-20251001
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-opus-4-5-20251101
litellm_params:
model: anthropic/claude-opus-4-5-20251101
api_key: os.environ/ANTHROPIC_API_KEY
```
**Warning:** These default model names may change with new Claude Code versions. Check LiteLLM proxy logs for "model not found" errors to identify what Claude Code is requesting.
## Additional Resources
- [LiteLLM Documentation](https://docs.litellm.ai/)
- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)

134
cookbook/ai_coding_tool_guides/index.json Normal file
View file

@ -0,0 +1,134 @@
[{
"title": "Claude Code Quickstart",
"description": "This is a quickstart guide to using Claude Code with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api",
"date": "2026-01-15",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM"
]
},
{
"title": "Claude Code with MCPs",
"description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_mcp",
"date": "2026-01-15",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"MCP"
]
},
{
"title": "Claude Code with Non-Anthropic Models",
"description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"OpenAI",
"Gemini"
]
},
{
"title": "Cursor Quickstart",
"description": "This is a quickstart guide to using Cursor with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/cursor_integration",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Cursor",
"LiteLLM",
"Quickstart"
]
},
{
"title": "Github Copilot Quickstart",
"description": "This is a quickstart guide to using Github Copilot with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Github Copilot",
"LiteLLM",
"Quickstart"
]
},
{
"title": "LiteLLM Gemini CLI Quickstart",
"description": "This is a quickstart guide to using LiteLLM Gemini CLI.",
"url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"Gemini CLI",
"Gemini",
"LiteLLM",
"Quickstart"
]
},
{
"title": "OpenAI Codex CLI Quickstart",
"description": "This is a quickstart guide to using OpenAI Codex CLI.",
"url": "https://docs.litellm.ai/docs/tutorials/openai_codex",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"OpenAI Codex CLI",
"OpenAI",
"LiteLLM",
"Quickstart"
]
},
{
"title": "OpenWebUI Quickstart",
"description": "This is a quickstart guide to using OpenWebUI with LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/openweb_ui",
"date": "2026-01-16",
"version": "1.0.0",
"tags": [
"OpenWebUI",
"LiteLLM",
"Quickstart"
]
},
{
"title": "AI Coding Tool Usage Tracking",
"description": "This is a guide to tracking usage for AI coding tools monitor the use of Claude Code , Google Antigravity, OpenAI Codex, Roo Code etc. through LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/cost_tracking_coding",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"Gemini CLI",
"OpenAI Codex",
"LiteLLM"
]
},
{
"title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)",
"description": "This is a guide for using Web Search with Claude Code via LiteLLM.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM",
"Web Search"
]
},
{
"title": "Track Claude Code Usage per user via Custom Headers",
"description": "This is a guide for tracking claude code user usage by passing a customer ID header.",
"url": "https://docs.litellm.ai/docs/tutorials/claude_code_customer_tracking",
"date": "2026-01-17",
"version": "1.0.0",
"tags": [
"Claude Code",
"LiteLLM"
]
}]

144
cookbook/anthropic_agent_sdk/README.md Normal file
View file

@ -0,0 +1,144 @@
# Claude Agent SDK with LiteLLM Gateway
A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK.
## Quick Start
### 1. Install dependencies
```bash
pip install anthropic claude-agent-sdk litellm
```
### 2. Start LiteLLM proxy
```bash
# Simple start with Claude
litellm --model claude-sonnet-4-20250514
# Or with a config file
litellm --config config.yaml
```
### 3. Run the chat
**Basic Agent (no MCP):**
```bash
python main.py
```
**Agent with MCP (DeepWiki2 for research):**
```bash
python agent_with_mcp.py
```
If MCP connection fails, you can disable it:
```bash
USE_MCP=false python agent_with_mcp.py
```
That's it! You can now chat with the agent in your terminal.
### Chat Commands
While chatting, you can use these commands:
- `models` - List all available models (fetched from your LiteLLM proxy)
- `model` - Switch to a different model
- `clear` - Start a new conversation
- `quit` or `exit` - End the chat
The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured.
## Configuration
Set these environment variables if needed:
```bash
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
export LITELLM_MODEL="bedrock-claude-sonnet-4.5"
```
Or just use the defaults - it'll connect to `http://localhost:4000` by default.
## Files
- `main.py` - Basic interactive agent without MCP
- `agent_with_mcp.py` - Agent with MCP server integration (DeepWiki2)
- `common.py` - Shared utilities and functions
- `config.example.yaml` - Example LiteLLM configuration
- `requirements.txt` - Python dependencies
## Example Config File
If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`):
```yaml
model_list:
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
```
Then start LiteLLM with: `litellm --config config.yaml`
## How It Works
The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
```python
# Point to LiteLLM gateway (not Anthropic)
os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key
# Use any model configured in LiteLLM
options = ClaudeAgentOptions(
model="bedrock-claude-sonnet-4", # or gpt-4, or anything else
system_prompt="You are a helpful assistant.",
max_turns=50,
)
```
Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically.
## Why Use This?
- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc.
- **Cost tracking**: LiteLLM tracks spending across all your agent conversations
- **Rate limiting**: Set budgets and limits on your agent usage
- **Load balancing**: Distribute requests across multiple API keys or regions
- **Fallbacks**: Automatically retry with a different model if one fails
## Troubleshooting
**Connection errors?**
- Make sure LiteLLM is running: `litellm --model your-model`
- Check the URL is correct (default: `http://localhost:4000`)
**Authentication errors?**
- Verify your LiteLLM API key is correct
- Make sure the model is configured in your LiteLLM setup
**Model not found?**
- Check the model name matches what's in your LiteLLM config
- Run `litellm --model your-model` to test it works
**Agent with MCP stuck or failing?**
- The MCP server might not be available at `http://localhost:4000/mcp/deepwiki2`
- Try disabling MCP: `USE_MCP=false python agent_with_mcp.py`
- Or use the basic agent: `python main.py`
## Learn More
- [LiteLLM Docs](https://docs.litellm.ai/)
- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk)
- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start)

140
cookbook/anthropic_agent_sdk/agent_with_mcp.py Normal file
View file

@ -0,0 +1,140 @@
"""
Interactive Claude Agent SDK CLI with MCP Support
This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy,
with MCP (Model Context Protocol) server integration for enhanced capabilities.
"""
import asyncio
import os
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
from common import (
Config,
fetch_available_models,
setup_litellm_env,
print_header,
handle_model_list,
handle_model_switch,
stream_response,
)
async def interactive_chat_with_mcp():
"""
Interactive CLI chat with the agent and MCP server
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
current_model = config.LITELLM_MODEL
# MCP server configuration
mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
if not use_mcp:
print("⚠️ MCP disabled via USE_MCP=false")
print_header(litellm_base_url, current_model, has_mcp=use_mcp)
while True:
# Configure agent options
if use_mcp:
try:
# Try with MCP server (HTTP transport)
# Using McpHttpServerConfig format from Agent SDK
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant with access to DeepWiki for research. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
mcp_servers={
"deepwiki2": {
"type": "http",
"url": mcp_server_url,
"headers": {
"Authorization": f"Bearer {config.LITELLM_API_KEY}"
}
}
},
)
except Exception as e:
print(f"⚠️ Warning: Could not configure MCP server: {e}")
print("Continuing without MCP...\n")
use_mcp = False
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
else:
# Without MCP
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
# Create agent client
try:
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
user_input = input("\n👤 You: ").strip()
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)
except Exception as e:
print(f"\n❌ Error creating agent client: {e}")
print("This might be an MCP configuration issue. Try running without MCP:")
print(" USE_MCP=false python agent_with_mcp.py")
print("\nOr use the basic agent:")
print(" python main.py")
return
def main():
"""Run interactive chat with MCP"""
try:
asyncio.run(interactive_chat_with_mcp())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
if __name__ == "__main__":
main()

160
cookbook/anthropic_agent_sdk/common.py Normal file
View file

@ -0,0 +1,160 @@
"""
Common utilities for Claude Agent SDK examples
"""
import os
import httpx
class Config:
"""Configuration for LiteLLM Gateway connection"""
# LiteLLM proxy URL (default to local instance)
LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
# LiteLLM API key (master key or virtual key)
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
# Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
"""
Fetch available models from LiteLLM proxy /models endpoint
"""
try:
async with httpx.AsyncClient() as client:
response = await client.get(
f"{base_url}/models",
headers={"Authorization": f"Bearer {api_key}"},
timeout=10.0
)
response.raise_for_status()
data = response.json()
return [model["id"] for model in data.get("data", [])]
except Exception as e:
print(f"⚠️ Warning: Could not fetch models from proxy: {e}")
print("Using default model list...")
# Fallback to default models
return [
"bedrock-claude-sonnet-3.5",
"bedrock-claude-sonnet-4",
"bedrock-claude-sonnet-4.5",
"bedrock-claude-opus-4.5",
"bedrock-nova-premier",
]
def setup_litellm_env(config: Config):
"""
Configure environment variables to point Agent SDK to LiteLLM
"""
litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
return litellm_base_url
def print_header(base_url: str, current_model: str, has_mcp: bool = False):
"""
Print the chat header
"""
mcp_indicator = " + MCP" if has_mcp else ""
print("=" * 70)
print(f"🤖 Claude Agent SDK with LiteLLM Gateway{mcp_indicator} - Interactive Chat")
print("=" * 70)
print(f"🚀 Connected to: {base_url}")
print(f"📦 Current model: {current_model}")
if has_mcp:
print("🔌 MCP: deepwiki2 enabled")
print("\nType your messages below. Commands:")
print(" - 'quit' or 'exit' to end the conversation")
print(" - 'clear' to start a new conversation")
print(" - 'model' to switch models")
print(" - 'models' to list available models")
print("=" * 70)
print()
def handle_model_list(available_models: list[str], current_model: str):
"""
Display available models
"""
print("\n📋 Available models:")
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
"""
Handle model switching
Returns:
tuple: (new_model, should_restart_conversation)
"""
print("\n📋 Select a model:")
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
try:
choice = input("\nEnter number (or press Enter to cancel): ").strip()
if choice:
idx = int(choice) - 1
if 0 <= idx < len(available_models):
new_model = available_models[idx]
print(f"\n✅ Switched to: {new_model}")
print("🔄 Starting new conversation with new model...\n")
return new_model, True
else:
print("❌ Invalid choice")
except (ValueError, IndexError):
print("❌ Invalid input")
return current_model, False
async def stream_response(client, user_input: str):
"""
Stream response from the agent
"""
print("\n🤖 Assistant: ", end='', flush=True)
try:
await client.query(user_input)
# Show loading indicator
print("⏳ thinking...", end='', flush=True)
# Stream the response
first_chunk = True
async for msg in client.receive_response():
# Clear loading indicator on first message
if first_chunk:
print("\r🤖 Assistant: ", end='', flush=True)
first_chunk = False
# Handle different message types
if hasattr(msg, 'type'):
if msg.type == 'content_block_delta':
# Streaming text delta
if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
print(msg.delta.text, end='', flush=True)
elif msg.type == 'content_block_start':
# Start of content block
if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
print(msg.content_block.text, end='', flush=True)
# Fallback to original content handling
if hasattr(msg, 'content'):
for content_block in msg.content:
if hasattr(content_block, 'text'):
print(content_block.text, end='', flush=True)
print() # New line after response
except Exception as e:
print(f"\r\n❌ Error: {e}")
print("Please check your LiteLLM gateway is running and configured correctly.")

25
cookbook/anthropic_agent_sdk/config.example.yaml Normal file
View file

@ -0,0 +1,25 @@
model_list:
- model_name: bedrock-claude-sonnet-3.5
litellm_params:
model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-sonnet-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-claude-opus-4.5
litellm_params:
model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-nova-premier
litellm_params:
model: "bedrock/amazon.nova-premier-v1:0"
aws_region_name: "us-east-1"

95
cookbook/anthropic_agent_sdk/main.py Normal file
View file

@ -0,0 +1,95 @@
"""
Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway
This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy.
LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.)
through the Claude Agent SDK by pointing it to the LiteLLM gateway.
"""
import asyncio
from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
from common import (
Config,
fetch_available_models,
setup_litellm_env,
print_header,
handle_model_list,
handle_model_switch,
stream_response,
)
async def interactive_chat():
"""
Interactive CLI chat with the agent
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
current_model = config.LITELLM_MODEL
print_header(litellm_base_url, current_model)
while True:
# Configure agent options for each conversation
options = ClaudeAgentOptions(
system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
model=current_model,
max_turns=50,
)
# Create agent client
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
user_input = input("\n👤 You: ").strip()
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)
def main():
"""Run interactive chat"""
try:
asyncio.run(interactive_chat())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
if __name__ == "__main__":
main()

2
cookbook/anthropic_agent_sdk/requirements.txt Normal file
View file

@ -0,0 +1,2 @@
claude-agent-sdk
httpx>=0.27.0

4
cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py
View file

@ -1,16 +1,12 @@
import sys, os
import traceback
from dotenv import load_dotenv
load_dotenv()
import litellm
from litellm import embedding, completion, completion_cost
from autoevals.llm import *
###################
import litellm
# litellm completion call
question = "which country has the highest population"

21
cookbook/codellama-server/main.py
View file

@ -1,11 +1,12 @@
import traceback
from flask import Flask, request, jsonify, abort, Response
from flask import Flask, request, Response
from flask_cors import CORS
import traceback
import litellm
from util import handle_error
from litellm import completion
import os, dotenv, time
import os
import dotenv
import time
import json
dotenv.load_dotenv()
@ -20,9 +21,9 @@ verbose = True
# litellm.caching_with_models = True # CACHING: caching_with_models Keys in the cache are messages + model. - to learn more: https://docs.litellm.ai/docs/caching/
######### PROMPT LOGGING ##########
os.environ[
"PROMPTLAYER_API_KEY"
] = "" # set your promptlayer key here - https://promptlayer.com/
os.environ["PROMPTLAYER_API_KEY"] = (
"" # set your promptlayer key here - https://promptlayer.com/
)
# set callbacks
litellm.success_callback = ["promptlayer"]
@ -57,9 +58,9 @@ def api_completion():
try:
if "prompt" not in data:
raise ValueError("data needs to have prompt")
data[
"model"
] = "togethercomputer/CodeLlama-34b-Instruct" # by default use Together AI's CodeLlama model - https://api.together.xyz/playground/chat?model=togethercomputer%2FCodeLlama-34b-Instruct
data["model"] = (
"togethercomputer/CodeLlama-34b-Instruct" # by default use Together AI's CodeLlama model - https://api.together.xyz/playground/chat?model=togethercomputer%2FCodeLlama-34b-Instruct
)
# COMPLETION CALL
system_prompt = "Only respond to questions about code. Say 'I don't know' to anything outside of that."
messages = [
@ -75,7 +76,7 @@ def api_completion():
"stream" in data and data["stream"] == True
): # use generate_responses to stream responses
return Response(data_generator(response), mimetype="text/event-stream")
except Exception as e:
except Exception:
# call handle_error function
print_verbose(f"Got Error api_completion(): {traceback.format_exc()}")
## LOG FAILURE

1
cookbook/community-resources/get_hf_models.py
View file

@ -1,5 +1,4 @@
import requests
from urllib.parse import urlparse, parse_qs
def get_next_url(response):

412
cookbook/google_adk_litellm_tutorial.ipynb vendored Normal file
View file

@ -0,0 +1,412 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "7aa8875d",
"metadata": {},
"source": [
"# Google ADK with LiteLLM\n",
"\n",
"Use Google ADK with LiteLLM Python SDK, LiteLLM Proxy.\n",
"\n",
"This tutorial shows you how to create intelligent agents using Agent Development Kit (ADK) with support for multiple Large Language Model (LLM) providers through LiteLLM."
]
},
{
"cell_type": "markdown",
"id": "a4d249c3",
"metadata": {},
"source": [
"## Overview\n",
"\n",
"ADK (Agent Development Kit) allows you to build intelligent agents powered by LLMs. By integrating with LiteLLM, you can:\n",
"\n",
"- Use multiple LLM providers (OpenAI, Anthropic, Google, etc.)\n",
"- Switch easily between models from different providers\n",
"- Connect to a LiteLLM proxy for centralized model management"
]
},
{
"cell_type": "markdown",
"id": "a0bbb56b",
"metadata": {},
"source": [
"## Prerequisites\n",
"\n",
"- Python environment setup\n",
"- API keys for model providers (OpenAI, Anthropic, Google AI Studio)\n",
"- Basic understanding of LLMs and agent concepts"
]
},
{
"cell_type": "markdown",
"id": "7fee50a8",
"metadata": {},
"source": [
"## Installation"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "44106a23",
"metadata": {},
"outputs": [],
"source": [
"# Install dependencies\n",
"!pip install google-adk litellm"
]
},
{
"cell_type": "markdown",
"id": "2171740a",
"metadata": {},
"source": [
"## 1. Setting Up Environment"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6695807e",
"metadata": {},
"outputs": [],
"source": [
"# Setup environment and API keys\n",
"import os\n",
"import asyncio\n",
"from google.adk.agents import Agent\n",
"from google.adk.models.lite_llm import LiteLlm # For multi-model support\n",
"from google.adk.sessions import InMemorySessionService\n",
"from google.adk.runners import Runner\n",
"from google.genai import types\n",
"import litellm # Import for proxy configuration\n",
"\n",
"# Set your API keys\n",
"os.environ['GOOGLE_API_KEY'] = 'your-google-api-key' # For Gemini models\n",
"os.environ['OPENAI_API_KEY'] = 'your-openai-api-key' # For OpenAI models\n",
"os.environ['ANTHROPIC_API_KEY'] = 'your-anthropic-api-key' # For Claude models\n",
"\n",
"# Define model constants for cleaner code\n",
"MODEL_GEMINI_PRO = 'gemini-1.5-pro'\n",
"MODEL_GPT_4O = 'openai/gpt-4o'\n",
"MODEL_CLAUDE_SONNET = 'anthropic/claude-3-sonnet-20240229'"
]
},
{
"cell_type": "markdown",
"id": "d2b1ed59",
"metadata": {},
"source": [
"## 2. Define a Simple Tool"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "04b3ef5b",
"metadata": {},
"outputs": [],
"source": [
"# Weather tool implementation\n",
"def get_weather(city: str) -> dict:\n",
" \"\"\"Retrieves the current weather report for a specified city.\"\"\"\n",
" print(f'Tool: get_weather called for city: {city}')\n",
"\n",
" # Mock weather data\n",
" mock_weather_db = {\n",
" 'newyork': {\n",
" 'status': 'success',\n",
" 'report': 'The weather in New York is sunny with a temperature of 25°C.'\n",
" },\n",
" 'london': {\n",
" 'status': 'success',\n",
" 'report': \"It's cloudy in London with a temperature of 15°C.\"\n",
" },\n",
" 'tokyo': {\n",
" 'status': 'success',\n",
" 'report': 'Tokyo is experiencing light rain and a temperature of 18°C.'\n",
" },\n",
" }\n",
"\n",
" city_normalized = city.lower().replace(' ', '')\n",
"\n",
" if city_normalized in mock_weather_db:\n",
" return mock_weather_db[city_normalized]\n",
" else:\n",
" return {\n",
" 'status': 'error',\n",
" 'error_message': f\"Sorry, I don't have weather information for '{city}'.\"\n",
" }"
]
},
{
"cell_type": "markdown",
"id": "727b15c9",
"metadata": {},
"source": [
"## 3. Helper Function for Agent Interaction"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f77449bf",
"metadata": {},
"outputs": [],
"source": [
"# Agent interaction helper function\n",
"async def call_agent_async(query: str, runner, user_id, session_id):\n",
" \"\"\"Sends a query to the agent and prints the final response.\"\"\"\n",
" print(f'\\n>>> User Query: {query}')\n",
"\n",
" content = types.Content(role='user', parts=[types.Part(text=query)])\n",
" final_response_text = 'Agent did not produce a final response.'\n",
"\n",
" async for event in runner.run_async(\n",
" user_id=user_id,\n",
" session_id=session_id,\n",
" new_message=content\n",
" ):\n",
" if event.is_final_response():\n",
" if event.content and event.content.parts:\n",
" final_response_text = event.content.parts[0].text\n",
" break\n",
" print(f'<<< Agent Response: {final_response_text}')"
]
},
{
"cell_type": "markdown",
"id": "0ac87987",
"metadata": {},
"source": [
"## 4. Using Different Model Providers with ADK\n",
"\n",
"### 4.1 Using OpenAI Models"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e167d557",
"metadata": {},
"outputs": [],
"source": [
"# OpenAI model implementation\n",
"weather_agent_gpt = Agent(\n",
" name='weather_agent_gpt',\n",
" model=LiteLlm(model=MODEL_GPT_4O),\n",
" description='Provides weather information using OpenAI\\'s GPT.',\n",
" instruction=(\n",
" 'You are a helpful weather assistant powered by GPT-4o. '\n",
" \"Use the 'get_weather' tool for city weather requests. \"\n",
" 'Present information clearly.'\n",
" ),\n",
" tools=[get_weather],\n",
")\n",
"\n",
"session_service_gpt = InMemorySessionService()\n",
"session_gpt = session_service_gpt.create_session(\n",
" app_name='weather_app', user_id='user_1', session_id='session_gpt'\n",
")\n",
"\n",
"runner_gpt = Runner(\n",
" agent=weather_agent_gpt,\n",
" app_name='weather_app',\n",
" session_service=session_service_gpt,\n",
")\n",
"\n",
"async def test_gpt_agent():\n",
" print('\\n--- Testing GPT Agent ---')\n",
" await call_agent_async(\n",
" \"What's the weather in London?\",\n",
" runner=runner_gpt,\n",
" user_id='user_1',\n",
" session_id='session_gpt',\n",
" )\n",
"\n",
"# To execute in a notebook cell:\n",
"# await test_gpt_agent()"
]
},
{
"cell_type": "markdown",
"id": "f9cb0613",
"metadata": {},
"source": [
"### 4.2 Using Anthropic Models"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1c653665",
"metadata": {},
"outputs": [],
"source": [
"# Anthropic model implementation\n",
"weather_agent_claude = Agent(\n",
" name='weather_agent_claude',\n",
" model=LiteLlm(model=MODEL_CLAUDE_SONNET),\n",
" description='Provides weather information using Anthropic\\'s Claude.',\n",
" instruction=(\n",
" 'You are a helpful weather assistant powered by Claude Sonnet. '\n",
" \"Use the 'get_weather' tool for city weather requests. \"\n",
" 'Present information clearly.'\n",
" ),\n",
" tools=[get_weather],\n",
")\n",
"\n",
"session_service_claude = InMemorySessionService()\n",
"session_claude = session_service_claude.create_session(\n",
" app_name='weather_app', user_id='user_1', session_id='session_claude'\n",
")\n",
"\n",
"runner_claude = Runner(\n",
" agent=weather_agent_claude,\n",
" app_name='weather_app',\n",
" session_service=session_service_claude,\n",
")\n",
"\n",
"async def test_claude_agent():\n",
" print('\\n--- Testing Claude Agent ---')\n",
" await call_agent_async(\n",
" \"What's the weather in Tokyo?\",\n",
" runner=runner_claude,\n",
" user_id='user_1',\n",
" session_id='session_claude',\n",
" )\n",
"\n",
"# To execute in a notebook cell:\n",
"# await test_claude_agent()"
]
},
{
"cell_type": "markdown",
"id": "bf9d863b",
"metadata": {},
"source": [
"### 4.3 Using Google's Gemini Models"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "83f49d0a",
"metadata": {},
"outputs": [],
"source": [
"# Gemini model implementation\n",
"weather_agent_gemini = Agent(\n",
" name='weather_agent_gemini',\n",
" model=MODEL_GEMINI_PRO,\n",
" description='Provides weather information using Google\\'s Gemini.',\n",
" instruction=(\n",
" 'You are a helpful weather assistant powered by Gemini Pro. '\n",
" \"Use the 'get_weather' tool for city weather requests. \"\n",
" 'Present information clearly.'\n",
" ),\n",
" tools=[get_weather],\n",
")\n",
"\n",
"session_service_gemini = InMemorySessionService()\n",
"session_gemini = session_service_gemini.create_session(\n",
" app_name='weather_app', user_id='user_1', session_id='session_gemini'\n",
")\n",
"\n",
"runner_gemini = Runner(\n",
" agent=weather_agent_gemini,\n",
" app_name='weather_app',\n",
" session_service=session_service_gemini,\n",
")\n",
"\n",
"async def test_gemini_agent():\n",
" print('\\n--- Testing Gemini Agent ---')\n",
" await call_agent_async(\n",
" \"What's the weather in New York?\",\n",
" runner=runner_gemini,\n",
" user_id='user_1',\n",
" session_id='session_gemini',\n",
" )\n",
"\n",
"# To execute in a notebook cell:\n",
"# await test_gemini_agent()"
]
},
{
"cell_type": "markdown",
"id": "93bc5fd0",
"metadata": {},
"source": [
"## 5. Using LiteLLM Proxy with ADK"
]
},
{
"cell_type": "markdown",
"id": "b4275151",
"metadata": {},
"source": [
"| Variable | Description |\n",
"|----------|-------------|\n",
"| `LITELLM_PROXY_API_KEY` | The API key for the LiteLLM proxy |\n",
"| `LITELLM_PROXY_API_BASE` | The base URL for the LiteLLM proxy |\n",
"| `USE_LITELLM_PROXY` or `litellm.use_litellm_proxy` | When set to True, your request will be sent to LiteLLM proxy. |"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "256530a6",
"metadata": {},
"outputs": [],
"source": [
"# LiteLLM proxy integration\n",
"os.environ['LITELLM_PROXY_API_KEY'] = 'your-litellm-proxy-api-key'\n",
"os.environ['LITELLM_PROXY_API_BASE'] = 'your-litellm-proxy-url' # e.g., 'http://localhost:4000'\n",
"litellm.use_litellm_proxy = True\n",
"\n",
"weather_agent_proxy_env = Agent(\n",
" name='weather_agent_proxy_env',\n",
" model=LiteLlm(model='gpt-4o'),\n",
" description='Provides weather information using a model from LiteLLM proxy.',\n",
" instruction=(\n",
" 'You are a helpful weather assistant. '\n",
" \"Use the 'get_weather' tool for city weather requests. \"\n",
" 'Present information clearly.'\n",
" ),\n",
" tools=[get_weather],\n",
")\n",
"\n",
"session_service_proxy_env = InMemorySessionService()\n",
"session_proxy_env = session_service_proxy_env.create_session(\n",
" app_name='weather_app', user_id='user_1', session_id='session_proxy_env'\n",
")\n",
"\n",
"runner_proxy_env = Runner(\n",
" agent=weather_agent_proxy_env,\n",
" app_name='weather_app',\n",
" session_service=session_service_proxy_env,\n",
")\n",
"\n",
"async def test_proxy_env_agent():\n",
" print('\\n--- Testing Proxy-enabled Agent (Environment Variables) ---')\n",
" await call_agent_async(\n",
" \"What's the weather in London?\",\n",
" runner=runner_proxy_env,\n",
" user_id='user_1',\n",
" session_id='session_proxy_env',\n",
" )\n",
"\n",
"# To execute in a notebook cell:\n",
"# await test_proxy_env_agent()"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

378
cookbook/liteLLM_Baseten.ipynb vendored
View file

@ -1,238 +1,146 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "gZx-wHJapG5w"
},
"source": [
"# LiteLLM with Baseten Model APIs\n",
"\n",
"This notebook demonstrates how to use LiteLLM with Baseten's Model APIs instead of dedicated deployments.\n",
"\n",
"## Example Usage\n",
"```python\n",
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n",
" max_tokens=1000,\n",
" temperature=0.7\n",
")\n",
"```\n",
"\n",
"## Setup"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# Use liteLLM to call Falcon, Wizard, MPT 7B using OpenAI chatGPT Input/output\n",
"\n",
"* Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
"* Wizard LM: https://app.baseten.co/explore/wizardlm\n",
"* MPT 7B Base: https://app.baseten.co/explore/mpt_7b_instruct\n",
"\n",
"\n",
"## Call all baseten llm models using OpenAI chatGPT Input/Output using liteLLM\n",
"Example call\n",
"```python\n",
"model = \"q841o8w\" # baseten model version ID\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"```"
],
"metadata": {
"id": "gZx-wHJapG5w"
}
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "4JSRa0QVogPo"
},
"outputs": [],
"source": [
"%pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "VEukLhDzo4vw"
},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion\n",
"\n",
"# Set your Baseten API key\n",
"os.environ['BASETEN_API_KEY'] = \"\" #@param {type:\"string\"}\n",
"\n",
"# Test message\n",
"messages = [{\"role\": \"user\", \"content\": \"What is AGI?\"}]"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "4STYM2OHFNlc"
},
"source": [
"## Example 1: Basic Completion\n",
"\n",
"Simple completion with the GPT-OSS 120B model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "DorpLxw1FHbC"
},
"outputs": [],
"source": [
"print(\"=== Basic Completion ===\")\n",
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=messages,\n",
" max_tokens=1000,\n",
" temperature=0.7,\n",
" top_p=0.9,\n",
" presence_penalty=0.1,\n",
" frequency_penalty=0.1,\n",
")\n",
"print(f\"Response: {response.choices[0].message.content}\")\n",
"print(f\"Usage: {response.usage}\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "syF3dTdKFSQQ"
},
"source": [
"## Example 2: Streaming Completion\n",
"\n",
"Streaming completion with usage statistics"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "4JSRa0QVogPo"
},
"outputs": [],
"source": [
"!pip install litellm==0.1.399\n",
"!pip install baseten urllib3"
]
},
{
"cell_type": "code",
"source": [
"import os\n",
"import litellm\n",
"from litellm import completion"
],
"metadata": {
"id": "VEukLhDzo4vw"
},
"execution_count": 2,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Setup"
],
"metadata": {
"id": "4STYM2OHFNlc"
}
},
{
"cell_type": "code",
"source": [
"os.environ['BASETEN_API_KEY'] = \"\" #@param\n",
"messages = [{ \"content\": \"what does Baseten do? \",\"role\": \"user\"}]"
],
"metadata": {
"id": "DorpLxw1FHbC"
},
"execution_count": 21,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Calling Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
"### Pass Your Baseten model `Version ID` as `model`"
],
"metadata": {
"id": "syF3dTdKFSQQ"
}
},
{
"cell_type": "code",
"source": [
"model = \"qvv0xeq\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "rPgSoMlsojz0",
"outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050"
},
"execution_count": 18,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': \"what does Baseten do? \\nI'm sorry, I cannot provide a specific answer as\"}}],\n",
" 'created': 1692135883.699066,\n",
" 'model': 'qvv0xeq'}"
]
},
"metadata": {},
"execution_count": 18
}
]
},
{
"cell_type": "markdown",
"source": [
"## Calling Wizard LM https://app.baseten.co/explore/wizardlm\n",
"### Pass Your Baseten model `Version ID` as `model`"
],
"metadata": {
"id": "7n21UroEGCGa"
}
},
{
"cell_type": "code",
"source": [
"model = \"q841o8w\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "uLVWFH899lAF",
"outputId": "61c2bc74-673b-413e-bb40-179cf408523d"
},
"execution_count": 19,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': 'As an AI language model, I do not have personal beliefs or practices, but based on the information available online, Baseten is a popular name for a traditional Ethiopian dish made with injera, a spongy flatbread, and wat, a spicy stew made with meat or vegetables. It is typically served for breakfast or dinner and is a staple in Ethiopian cuisine. The name Baseten is also used to refer to a traditional Ethiopian coffee ceremony, where coffee is brewed and served in a special ceremony with music and food.'}}],\n",
" 'created': 1692135900.2806294,\n",
" 'model': 'q841o8w'}"
]
},
"metadata": {},
"execution_count": 19
}
]
},
{
"cell_type": "markdown",
"source": [
"## Calling mosaicml/mpt-7b https://app.baseten.co/explore/mpt_7b_instruct\n",
"### Pass Your Baseten model `Version ID` as `model`"
],
"metadata": {
"id": "6-TFwmPAGPXq"
}
},
{
"cell_type": "code",
"source": [
"model = \"31dxrj3\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "gbeYZOrUE_Bp",
"outputId": "838d86ea-2143-4cb3-bc80-2acc2346c37a"
},
"execution_count": 20,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': \"\\n===================\\n\\nIt's a tool to build a local version of a game on your own machine to host\\non your website.\\n\\nIt's used to make game demos and show them on Twitter, Tumblr, and Facebook.\\n\\n\\n\\n## What's built\\n\\n- A directory of all your game directories, named with a version name and build number, with images linked to.\\n- Includes HTML to include in another site.\\n- Includes images for your icons and\"}}],\n",
" 'created': 1692135914.7472186,\n",
" 'model': '31dxrj3'}"
]
},
"metadata": {},
"execution_count": 20
}
]
}
]
}
"id": "rPgSoMlsojz0",
"outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050"
},
"outputs": [],
"source": [
"print(\"=== Streaming Completion ===\")\n",
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=[{\"role\": \"user\", \"content\": \"Write a short poem about AI\"}],\n",
" stream=True,\n",
" max_tokens=500,\n",
" temperature=0.8,\n",
" stream_options={\n",
" \"include_usage\": True,\n",
" \"continuous_usage_stats\": True\n",
" },\n",
")\n",
"\n",
"print(\"Streaming response:\")\n",
"for chunk in response:\n",
" if chunk.choices and chunk.choices[0].delta.content:\n",
" print(chunk.choices[0].delta.content, end=\"\", flush=True)\n",
"print(\"\\n\")"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

300
cookbook/liteLLM_IBM_Watsonx.ipynb vendored Normal file
View file

File diff suppressed because one or more lines are too long

382
cookbook/liteLLM_Langchain_Demo.ipynb vendored
View file

@ -1,201 +1,195 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "5hwntUxTMxEk"
},
"source": [
"# Langchain liteLLM Demo Notebook\n",
"## Use `ChatLiteLLM()` to instantly support 50+ LLM models\n",
"Langchain Docs: https://python.langchain.com/docs/integrations/chat/litellm\n",
"\n",
"Call all LLM models using the same I/O interface\n",
"\n",
"Example usage\n",
"```python\n",
"ChatLiteLLM(model=\"gpt-3.5-turbo\")\n",
"ChatLiteLLM(model=\"claude-2\", temperature=0.3)\n",
"ChatLiteLLM(model=\"command-nightly\")\n",
"ChatLiteLLM(model=\"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\")\n",
"```"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# Langchain liteLLM Demo Notebook\n",
"## Use `ChatLiteLLM()` to instantly support 50+ LLM models\n",
"Langchain Docs: https://python.langchain.com/docs/integrations/chat/litellm\n",
"\n",
"Call all LLM models using the same I/O interface\n",
"\n",
"Example usage\n",
"```python\n",
"ChatLiteLLM(model=\"gpt-3.5-turbo\")\n",
"ChatLiteLLM(model=\"claude-2\", temperature=0.3)\n",
"ChatLiteLLM(model=\"command-nightly\")\n",
"ChatLiteLLM(model=\"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\")\n",
"```"
],
"metadata": {
"id": "5hwntUxTMxEk"
}
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "aPNAUsCvB6Sv"
},
"outputs": [],
"source": [
"!pip install litellm langchain"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "MOhRaVnhB-0J"
},
"outputs": [],
"source": [
"import os\n",
"from langchain.chat_models import ChatLiteLLM\n",
"from langchain.schema import HumanMessage"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "TahkCtlmCD65",
"outputId": "5ddda40f-f252-4830-a8d6-bd3fa68ae487"
},
"outputs": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "aPNAUsCvB6Sv"
},
"outputs": [],
"source": [
"!pip install litellm langchain"
]
},
{
"cell_type": "code",
"source": [
"import os\n",
"from langchain.chat_models import ChatLiteLLM\n",
"from langchain.prompts.chat import (\n",
" ChatPromptTemplate,\n",
" SystemMessagePromptTemplate,\n",
" AIMessagePromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
")\n",
"from langchain.schema import AIMessage, HumanMessage, SystemMessage"
],
"metadata": {
"id": "MOhRaVnhB-0J"
},
"execution_count": 2,
"outputs": []
},
{
"cell_type": "code",
"source": [
"os.environ['OPENAI_API_KEY'] = \"\"\n",
"chat = ChatLiteLLM(model=\"gpt-3.5-turbo\")\n",
"messages = [\n",
" HumanMessage(\n",
" content=\"what model are you\"\n",
" )\n",
"]\n",
"chat(messages)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "TahkCtlmCD65",
"outputId": "5ddda40f-f252-4830-a8d6-bd3fa68ae487"
},
"execution_count": 17,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"AIMessage(content='I am an AI model known as GPT-3, developed by OpenAI.', additional_kwargs={}, example=False)"
]
},
"metadata": {},
"execution_count": 17
}
]
},
{
"cell_type": "code",
"source": [
"os.environ['ANTHROPIC_API_KEY'] = \"\"\n",
"chat = ChatLiteLLM(model=\"claude-2\", temperature=0.3)\n",
"messages = [\n",
" HumanMessage(\n",
" content=\"what model are you\"\n",
" )\n",
"]\n",
"chat(messages)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "uXNDyU4jChcs",
"outputId": "bd74b4c6-f9fb-42dc-fdc3-9240d50503ba"
},
"execution_count": 23,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"AIMessage(content=\" I'm Claude, an AI assistant created by Anthropic.\", additional_kwargs={}, example=False)"
]
},
"metadata": {},
"execution_count": 23
}
]
},
{
"cell_type": "code",
"source": [
"os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
"chat = ChatLiteLLM(model=\"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\")\n",
"messages = [\n",
" HumanMessage(\n",
" content=\"what model are you?\"\n",
" )\n",
"]\n",
"chat(messages)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "czbDJRKcC7BV",
"outputId": "892e147d-831e-4884-dc71-040f92c3fb8e"
},
"execution_count": 27,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"AIMessage(content=\" I'm an AI based based on LLaMA models (LLaMA: Open and Efficient Foundation Language Models, Touvron et al. 2023), my knowledge was built from a massive corpus of text, including books, articles, and websites, and I was trained using a variety of machine learning algorithms. My model architecture is based on the transformer architecture, which is particularly well-suited for natural language processing tasks. My team of developers and I are constantly working to improve and fine-tune my performance, and I am always happy to help with any questions you may have!\", additional_kwargs={}, example=False)"
]
},
"metadata": {},
"execution_count": 27
}
]
},
{
"cell_type": "code",
"source": [
"os.environ['COHERE_API_KEY'] = \"\"\n",
"chat = ChatLiteLLM(model=\"command-nightly\")\n",
"messages = [\n",
" HumanMessage(\n",
" content=\"what model are you?\"\n",
" )\n",
"]\n",
"chat(messages)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "tZxpq5PDDY9Y",
"outputId": "7e86f4ed-ac7a-45e1-87d0-217da6cad666"
},
"execution_count": 30,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"AIMessage(content=' I am an AI-based large language model, or Chatbot, built by the company Cohere. I am designed to have polite, helpful, inclusive conversations with users. I am always learning and improving, and I am constantly being updated with new information and improvements.\\n\\nI am currently in the development phase, and I am not yet available to the general public. However, I am currently being used by a select group of users for testing and feedback.\\n\\nI am a large language model, which means that I am trained on a massive amount of data and can understand and respond to a wide range of requests and questions. I am also designed to be flexible and adaptable, so I can be customized to suit the needs of different users and use cases.\\n\\nI am currently being used to develop a range of applications, including customer service chatbots, content generation tools, and language translation services. I am also being used to train other language models and to develop new ways of using large language models.\\n\\nI am constantly being updated with new information and improvements, so I am always learning and improving. I am also being used to develop new ways of using large language models, so I am always evolving and adapting to new use cases and requirements.', additional_kwargs={}, example=False)"
]
},
"metadata": {},
"execution_count": 30
}
"data": {
"text/plain": [
"AIMessage(content='I am an AI model known as GPT-3, developed by OpenAI.', additional_kwargs={}, example=False)"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
]
],
"source": [
"os.environ['OPENAI_API_KEY'] = \"\"\n",
"chat = ChatLiteLLM(model=\"gpt-3.5-turbo\")\n",
"messages = [\n",
" HumanMessage(\n",
" content=\"what model are you\"\n",
" )\n",
"]\n",
"chat(messages)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "uXNDyU4jChcs",
"outputId": "bd74b4c6-f9fb-42dc-fdc3-9240d50503ba"
},
"outputs": [
{
"data": {
"text/plain": [
"AIMessage(content=\" I'm Claude, an AI assistant created by Anthropic.\", additional_kwargs={}, example=False)"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"os.environ['ANTHROPIC_API_KEY'] = \"\"\n",
"chat = ChatLiteLLM(model=\"claude-2\", temperature=0.3)\n",
"messages = [\n",
" HumanMessage(\n",
" content=\"what model are you\"\n",
" )\n",
"]\n",
"chat(messages)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "czbDJRKcC7BV",
"outputId": "892e147d-831e-4884-dc71-040f92c3fb8e"
},
"outputs": [
{
"data": {
"text/plain": [
"AIMessage(content=\" I'm an AI based based on LLaMA models (LLaMA: Open and Efficient Foundation Language Models, Touvron et al. 2023), my knowledge was built from a massive corpus of text, including books, articles, and websites, and I was trained using a variety of machine learning algorithms. My model architecture is based on the transformer architecture, which is particularly well-suited for natural language processing tasks. My team of developers and I are constantly working to improve and fine-tune my performance, and I am always happy to help with any questions you may have!\", additional_kwargs={}, example=False)"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
"chat = ChatLiteLLM(model=\"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\")\n",
"messages = [\n",
" HumanMessage(\n",
" content=\"what model are you?\"\n",
" )\n",
"]\n",
"chat(messages)"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "tZxpq5PDDY9Y",
"outputId": "7e86f4ed-ac7a-45e1-87d0-217da6cad666"
},
"outputs": [
{
"data": {
"text/plain": [
"AIMessage(content=' I am an AI-based large language model, or Chatbot, built by the company Cohere. I am designed to have polite, helpful, inclusive conversations with users. I am always learning and improving, and I am constantly being updated with new information and improvements.\\n\\nI am currently in the development phase, and I am not yet available to the general public. However, I am currently being used by a select group of users for testing and feedback.\\n\\nI am a large language model, which means that I am trained on a massive amount of data and can understand and respond to a wide range of requests and questions. I am also designed to be flexible and adaptable, so I can be customized to suit the needs of different users and use cases.\\n\\nI am currently being used to develop a range of applications, including customer service chatbots, content generation tools, and language translation services. I am also being used to train other language models and to develop new ways of using large language models.\\n\\nI am constantly being updated with new information and improvements, so I am always learning and improving. I am also being used to develop new ways of using large language models, so I am always evolving and adapting to new use cases and requirements.', additional_kwargs={}, example=False)"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"os.environ['COHERE_API_KEY'] = \"\"\n",
"chat = ChatLiteLLM(model=\"command-nightly\")\n",
"messages = [\n",
" HumanMessage(\n",
" content=\"what model are you?\"\n",
" )\n",
"]\n",
"chat(messages)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

2
cookbook/liteLLM_VertextAI_Example.ipynb vendored
View file

@ -43,7 +43,7 @@
"source": [
"# set you Vertex AI configs\n",
"import litellm\n",
"from litellm import embedding, completion\n",
"from litellm import completion\n",
"\n",
"litellm.vertex_project = \"hardy-device-386718\"\n",
"litellm.vertex_location = \"us-central1\""

187
cookbook/liteLLM_clarifai_Demo.ipynb vendored Normal file
View file

@ -0,0 +1,187 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# LiteLLM Clarifai \n",
"This notebook walks you through on how to use liteLLM integration of Clarifai and call LLM model from clarifai with response in openAI output format."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Pre-Requisites"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#install necessary packages\n",
"!pip install litellm\n",
"!pip install clarifai"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"To obtain Clarifai Personal Access Token follow the steps mentioned in the [link](https://docs.clarifai.com/clarifai-basics/authentication/personal-access-tokens/)"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"## Set Clarifai Credentials\n",
"import os\n",
"os.environ[\"CLARIFAI_API_KEY\"]= \"YOUR_CLARIFAI_PAT\" # Clarifai PAT"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Mistral-large"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import litellm\n",
"\n",
"litellm.set_verbose=False"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Mistral large response : ModelResponse(id='chatcmpl-6eed494d-7ae2-4870-b9c2-6a64d50a6151', choices=[Choices(finish_reason='stop', index=1, message=Message(content=\"In the grand tapestry of time, where tales unfold,\\nLies the chronicle of ages, a sight to behold.\\nA tale of empires rising, and kings of old,\\nOf civilizations lost, and stories untold.\\n\\nOnce upon a yesterday, in a time so vast,\\nHumans took their first steps, casting shadows in the past.\\nFrom the cradle of mankind, a journey they embarked,\\nThrough stone and bronze and iron, their skills they sharpened and marked.\\n\\nEgyptians built pyramids, reaching for the skies,\\nWhile Greeks sought wisdom, truth, in philosophies that lie.\\nRoman legions marched, their empire to expand,\\nAnd in the East, the Silk Road joined the world, hand in hand.\\n\\nThe Middle Ages came, with knights in shining armor,\\nFeudal lords and serfs, a time of both clamor and calm order.\\nThen Renaissance bloomed, like a flower in the sun,\\nA rebirth of art and science, a new age had begun.\\n\\nAcross the vast oceans, explorers sailed with courage bold,\\nDiscovering new lands, stories of adventure, untold.\\nIndustrial Revolution churned, progress in its wake,\\nMachines and factories, a whole new world to make.\\n\\nTwo World Wars raged, a testament to man's strife,\\nYet from the ashes rose hope, a renewed will for life.\\nInto the modern era, technology took flight,\\nConnecting every corner, bathed in digital light.\\n\\nHistory, a symphony, a melody of time,\\nA testament to human will, resilience so sublime.\\nIn every page, a lesson, in every tale, a guide,\\nFor understanding our past, shapes our future's tide.\", role='assistant'))], created=1713896412, model='https://api.clarifai.com/v2/users/mistralai/apps/completion/models/mistral-large/outputs', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=13, completion_tokens=338, total_tokens=351))\n"
]
}
],
"source": [
"from litellm import completion\n",
"\n",
"messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n",
"response=completion(\n",
" model=\"clarifai/mistralai.completion.mistral-large\",\n",
" messages=messages,\n",
" )\n",
"\n",
"print(f\"Mistral large response : {response}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Claude-2.1 "
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Claude-2.1 response : ModelResponse(id='chatcmpl-d126c919-4db4-4aa3-ac8f-7edea41e0b93', choices=[Choices(finish_reason='stop', index=1, message=Message(content=\" Here's a poem I wrote about history:\\n\\nThe Tides of Time\\n\\nThe tides of time ebb and flow,\\nCarrying stories of long ago.\\nFigures and events come into light,\\nShaping the future with all their might.\\n\\nKingdoms rise, empires fall, \\nLeaving traces that echo down every hall.\\nRevolutions bring change with a fiery glow,\\nToppling structures from long ago.\\n\\nExplorers traverse each ocean and land,\\nSeeking treasures they don't understand.\\nWhile artists and writers try to make their mark,\\nHoping their works shine bright in the dark.\\n\\nThe cycle repeats again and again,\\nAs humanity struggles to learn from its pain.\\nThough the players may change on history's stage,\\nThe themes stay the same from age to age.\\n\\nWar and peace, life and death,\\nLove and strife with every breath.\\nThe tides of time continue their dance,\\nAs we join in, by luck or by chance.\\n\\nSo we study the past to light the way forward, \\nHeeding warnings from stories told and heard.\\nThe future unfolds from this unending flow -\\nWhere the tides of time ultimately go.\", role='assistant'))], created=1713896579, model='https://api.clarifai.com/v2/users/anthropic/apps/completion/models/claude-2_1/outputs', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=12, completion_tokens=232, total_tokens=244))\n"
]
}
],
"source": [
"from litellm import completion\n",
"\n",
"messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n",
"response=completion(\n",
" model=\"clarifai/anthropic.completion.claude-2_1\",\n",
" messages=messages,\n",
" )\n",
"\n",
"print(f\"Claude-2.1 response : {response}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### OpenAI GPT-4 (Streaming)\n",
"Though clarifai doesn't support streaming, still you can call stream and get the response in standard StreamResponse format of liteLLM"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ModelResponse(id='chatcmpl-40ae19af-3bf0-4eb4-99f2-33aec3ba84af', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=\"In the quiet corners of time's grand hall,\\nLies the tale of rise and fall.\\nFrom ancient ruins to modern sprawl,\\nHistory, the greatest story of them all.\\n\\nEmpires have risen, empires have decayed,\\nThrough the eons, memories have stayed.\\nIn the book of time, history is laid,\\nA tapestry of events, meticulously displayed.\\n\\nThe pyramids of Egypt, standing tall,\\nThe Roman Empire's mighty sprawl.\\nFrom Alexander's conquest, to the Berlin Wall,\\nHistory, a silent witness to it all.\\n\\nIn the shadow of the past we tread,\\nWhere once kings and prophets led.\\nTheir stories in our hearts are spread,\\nEchoes of their words, in our minds are read.\\n\\nBattles fought and victories won,\\nActs of courage under the sun.\\nTales of love, of deeds done,\\nIn history's grand book, they all run.\\n\\nHeroes born, legends made,\\nIn the annals of time, they'll never fade.\\nTheir triumphs and failures all displayed,\\nIn the eternal march of history's parade.\\n\\nThe ink of the past is forever dry,\\nBut its lessons, we cannot deny.\\nIn its stories, truths lie,\\nIn its wisdom, we rely.\\n\\nHistory, a mirror to our past,\\nA guide for the future vast.\\nThrough its lens, we're ever cast,\\nIn the drama of life, forever vast.\", role='assistant', function_call=None, tool_calls=None), logprobs=None)], created=1714744515, model='https://api.clarifai.com/v2/users/openai/apps/chat-completion/models/GPT-4/outputs', object='chat.completion.chunk', system_fingerprint=None)\n",
"ModelResponse(id='chatcmpl-40ae19af-3bf0-4eb4-99f2-33aec3ba84af', choices=[StreamingChoices(finish_reason='stop', index=0, delta=Delta(content=None, role=None, function_call=None, tool_calls=None), logprobs=None)], created=1714744515, model='https://api.clarifai.com/v2/users/openai/apps/chat-completion/models/GPT-4/outputs', object='chat.completion.chunk', system_fingerprint=None)\n"
]
}
],
"source": [
"from litellm import completion\n",
"\n",
"messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n",
"response = completion(\n",
" model=\"clarifai/openai.chat-completion.GPT-4\",\n",
" messages=messages,\n",
" stream=True,\n",
" api_key = \"c75cc032415e45368be331fdd2c06db0\")\n",
"\n",
"for chunk in response:\n",
" print(chunk)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

650
cookbook/liteLLM_function_calling.ipynb vendored
View file

@ -1,331 +1,331 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "vnvlwUDZK7VA"
},
"source": [
"## Demo Notebook of Function Calling with liteLLM\n",
"- Supported Providers for Function Calling\n",
" - OpenAI - `gpt-4-0613` and `gpt-3.5-turbo-0613`\n",
"- In this notebook we use function calling with `litellm.completion()`"
]
},
"cells": [
{
"cell_type": "markdown",
"source": [
"## Demo Notebook of Function Calling with liteLLM\n",
"- Supported Providers for Function Calling\n",
" - OpenAI - `gpt-4-0613` and `gpt-3.5-turbo-0613`\n",
"- In this notebook we use function calling with `litellm.completion()`"
],
"metadata": {
"id": "vnvlwUDZK7VA"
}
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "KrINCwRfLgZV"
},
"outputs": [],
"source": [
"## Install liteLLM\n",
"!pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "nK7zR5OgLlh2"
},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"id": "dCQlyBxKLqbA"
},
"outputs": [],
"source": [
"os.environ['OPENAI_API_KEY'] = \"\" #@param"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "gfdGv-FMRCdX"
},
"source": [
"## Define Messages, Functions\n",
"We create a get_current_weather() function and pass that to GPT 3.5\n",
"\n",
"See OpenAI docs for this: https://openai.com/blog/function-calling-and-other-api-updates"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"id": "ERzsP1sfM19C"
},
"outputs": [],
"source": [
"messages = [\n",
" {\"role\": \"user\", \"content\": \"What is the weather like in Boston?\"}\n",
"]\n",
"\n",
"def get_current_weather(location):\n",
" if location == \"Boston, MA\":\n",
" return \"The weather is 12F\"\n",
"\n",
"functions = [\n",
" {\n",
" \"name\": \"get_current_weather\",\n",
" \"description\": \"Get the current weather in a given location\",\n",
" \"parameters\": {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"location\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The city and state, e.g. San Francisco, CA\"\n",
" },\n",
" \"unit\": {\n",
" \"type\": \"string\",\n",
" \"enum\": [\"celsius\", \"fahrenheit\"]\n",
" }\n",
" },\n",
" \"required\": [\"location\"]\n",
" }\n",
" }\n",
" ]"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "NX6by2VuRPnp"
},
"source": [
"## Call gpt-3.5-turbo-0613 to Decide what Function to call"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "QVoJ5PtxMlVx",
"outputId": "efe7a81f-e04a-4afc-aa60-a2b2648f5fb9"
},
"outputs": [
{
"cell_type": "code",
"source": [
"## Install liteLLM\n",
"!pip install litellm"
],
"metadata": {
"id": "KrINCwRfLgZV"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import os, litellm\n",
"from litellm import completion"
],
"metadata": {
"id": "nK7zR5OgLlh2"
},
"execution_count": 2,
"outputs": []
},
{
"cell_type": "code",
"source": [
"os.environ['OPENAI_API_KEY'] = \"\" #@param"
],
"metadata": {
"id": "dCQlyBxKLqbA"
},
"execution_count": 27,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Define Messages, Functions\n",
"We create a get_current_weather() function and pass that to GPT 3.5\n",
"\n",
"See OpenAI docs for this: https://openai.com/blog/function-calling-and-other-api-updates"
],
"metadata": {
"id": "gfdGv-FMRCdX"
}
},
{
"cell_type": "code",
"source": [
"messages = [\n",
" {\"role\": \"user\", \"content\": \"What is the weather like in Boston?\"}\n",
"]\n",
"\n",
"def get_current_weather(location):\n",
" if location == \"Boston, MA\":\n",
" return \"The weather is 12F\"\n",
"\n",
"functions = [\n",
" {\n",
" \"name\": \"get_current_weather\",\n",
" \"description\": \"Get the current weather in a given location\",\n",
" \"parameters\": {\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"location\": {\n",
" \"type\": \"string\",\n",
" \"description\": \"The city and state, e.g. San Francisco, CA\"\n",
" },\n",
" \"unit\": {\n",
" \"type\": \"string\",\n",
" \"enum\": [\"celsius\", \"fahrenheit\"]\n",
" }\n",
" },\n",
" \"required\": [\"location\"]\n",
" }\n",
" }\n",
" ]"
],
"metadata": {
"id": "ERzsP1sfM19C"
},
"execution_count": 25,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Call gpt-3.5-turbo-0613 to Decide what Function to call"
],
"metadata": {
"id": "NX6by2VuRPnp"
}
},
{
"cell_type": "code",
"source": [
"response = completion(model=\"gpt-3.5-turbo-0613\", messages=messages, functions=functions)\n",
"print(response)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "QVoJ5PtxMlVx",
"outputId": "efe7a81f-e04a-4afc-aa60-a2b2648f5fb9"
},
"execution_count": 9,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{\n",
" \"id\": \"chatcmpl-7mX4RiqdoislVEqfmfVjFSKp3hyIy\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1691801223,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": null,\n",
" \"function_call\": {\n",
" \"name\": \"get_current_weather\",\n",
" \"arguments\": \"{\\n \\\"location\\\": \\\"Boston, MA\\\"\\n}\"\n",
" }\n",
" },\n",
" \"finish_reason\": \"function_call\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 82,\n",
" \"completion_tokens\": 18,\n",
" \"total_tokens\": 100\n",
" }\n",
"}\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Parse GPT 3.5 Response\n",
"Read Information about what Function to Call"
],
"metadata": {
"id": "Yu0o2saDNLx8"
}
},
{
"cell_type": "code",
"source": [
"function_call_data = response[\"choices\"][0][\"message\"][\"function_call\"]\n",
"function_call_data"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "u1DzXLJsNOR5",
"outputId": "177e9501-0ce2-4619-9067-3047f18f6c79"
},
"execution_count": 11,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"<OpenAIObject at 0x7922c70ce930> JSON: {\n",
" \"name\": \"get_current_weather\",\n",
" \"arguments\": \"{\\n \\\"location\\\": \\\"Boston, MA\\\"\\n}\"\n",
"}"
]
},
"metadata": {},
"execution_count": 11
}
]
},
{
"cell_type": "code",
"source": [
"import json\n",
"function_name = function_call_data['name']\n",
"function_args = function_call_data['arguments']\n",
"function_args = json.loads(function_args)\n",
"print(function_name, function_args)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "tYb96Mh0NhH9",
"outputId": "13c4bb89-6f29-4b3b-afa7-302dcf2cdd5f"
},
"execution_count": 20,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"get_current_weather {'location': 'Boston, MA'}\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Call the get_current_weather() function"
],
"metadata": {
"id": "z3tstH_yN3fX"
}
},
{
"cell_type": "code",
"source": [
"if function_name == \"get_current_weather\":\n",
" result = get_current_weather(**function_args)\n",
" print(result)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "TSb8JHhgN5Zc",
"outputId": "ef140572-4020-4daf-ac8c-d5161be9aa5c"
},
"execution_count": 24,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"12F\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Send the response from get_current_weather back to the model to summarize"
],
"metadata": {
"id": "k4HGJE3NRmMI"
}
},
{
"cell_type": "code",
"source": [
"messages = [\n",
" {\"role\": \"user\", \"content\": \"What is the weather like in Boston?\"},\n",
" {\"role\": \"assistant\", \"content\": None, \"function_call\": {\"name\": \"get_current_weather\", \"arguments\": \"{ \\\"location\\\": \\\"Boston, MA\\\"}\"}},\n",
" {\"role\": \"function\", \"name\": \"get_current_weather\", \"content\": result}\n",
"]\n",
"response = completion(model=\"gpt-3.5-turbo-0613\", messages=messages, functions=functions)\n",
"print(response)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "a23cmEwiPaw7",
"outputId": "43259b86-0c4c-4fcb-eab7-6e1a788b2f21"
},
"execution_count": 26,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{\n",
" \"id\": \"chatcmpl-7mXGN62u75WXp1Lgen4iSgNvA7hHT\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1691801963,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"The current weather in Boston is 12 degrees Fahrenheit.\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 109,\n",
" \"completion_tokens\": 12,\n",
" \"total_tokens\": 121\n",
" }\n",
"}\n"
]
}
]
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"id\": \"chatcmpl-7mX4RiqdoislVEqfmfVjFSKp3hyIy\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1691801223,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": null,\n",
" \"function_call\": {\n",
" \"name\": \"get_current_weather\",\n",
" \"arguments\": \"{\\n \\\"location\\\": \\\"Boston, MA\\\"\\n}\"\n",
" }\n",
" },\n",
" \"finish_reason\": \"function_call\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 82,\n",
" \"completion_tokens\": 18,\n",
" \"total_tokens\": 100\n",
" }\n",
"}\n"
]
}
]
],
"source": [
"response = completion(model=\"gpt-3.5-turbo-0613\", messages=messages, functions=functions)\n",
"print(response)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Yu0o2saDNLx8"
},
"source": [
"## Parse GPT 3.5 Response\n",
"Read Information about what Function to Call"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "u1DzXLJsNOR5",
"outputId": "177e9501-0ce2-4619-9067-3047f18f6c79"
},
"outputs": [
{
"data": {
"text/plain": [
"<OpenAIObject at 0x7922c70ce930> JSON: {\n",
" \"name\": \"get_current_weather\",\n",
" \"arguments\": \"{\\n \\\"location\\\": \\\"Boston, MA\\\"\\n}\"\n",
"}"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"function_call_data = response[\"choices\"][0][\"message\"][\"function_call\"]\n",
"function_call_data"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "tYb96Mh0NhH9",
"outputId": "13c4bb89-6f29-4b3b-afa7-302dcf2cdd5f"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"get_current_weather {'location': 'Boston, MA'}\n"
]
}
],
"source": [
"import json\n",
"function_name = function_call_data['name']\n",
"function_args = function_call_data['arguments']\n",
"function_args = json.loads(function_args)\n",
"print(function_name, function_args)\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "z3tstH_yN3fX"
},
"source": [
"## Call the get_current_weather() function"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "TSb8JHhgN5Zc",
"outputId": "ef140572-4020-4daf-ac8c-d5161be9aa5c"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"12F\n"
]
}
],
"source": [
"if function_name == \"get_current_weather\":\n",
" result = get_current_weather(**function_args)\n",
" print(result)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "k4HGJE3NRmMI"
},
"source": [
"## Send the response from get_current_weather back to the model to summarize"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "a23cmEwiPaw7",
"outputId": "43259b86-0c4c-4fcb-eab7-6e1a788b2f21"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"id\": \"chatcmpl-7mXGN62u75WXp1Lgen4iSgNvA7hHT\",\n",
" \"object\": \"chat.completion\",\n",
" \"created\": 1691801963,\n",
" \"model\": \"gpt-3.5-turbo-0613\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"The current weather in Boston is 12 degrees Fahrenheit.\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"usage\": {\n",
" \"prompt_tokens\": 109,\n",
" \"completion_tokens\": 12,\n",
" \"total_tokens\": 121\n",
" }\n",
"}\n"
]
}
],
"source": [
"messages = [\n",
" {\"role\": \"user\", \"content\": \"What is the weather like in Boston?\"},\n",
" {\"role\": \"assistant\", \"content\": None, \"function_call\": {\"name\": \"get_current_weather\", \"arguments\": \"{ \\\"location\\\": \\\"Boston, MA\\\"}\"}},\n",
" {\"role\": \"function\", \"name\": \"get_current_weather\", \"content\": result}\n",
"]\n",
"response = completion(model=\"gpt-3.5-turbo-0613\", messages=messages, functions=functions)\n",
"print(response)"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

2
cookbook/litellm-ollama-docker-image/requirements.txt
View file

@ -1 +1 @@
litellm
litellm==1.61.15

4
cookbook/litellm-ollama-docker-image/test.py
View file

@ -1,13 +1,13 @@
import openai
api_base = f"http://0.0.0.0:8000"
api_base = "http://0.0.0.0:8000"
openai.api_base = api_base
openai.api_key = "temp-key"
print(openai.api_base)
print(f"LiteLLM: response from proxy with streaming")
print("LiteLLM: response from proxy with streaming")
response = openai.ChatCompletion.create(
model="ollama/llama2",
messages=[

1130
cookbook/litellm_Test_Multiple_Providers.ipynb vendored
View file

File diff suppressed because one or more lines are too long

97
cookbook/litellm_model_fallback.ipynb vendored
View file

@ -1,52 +1,51 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "j6yJsCGeaq8G"
},
"outputs": [],
"source": [
"!pip install litellm"
]
},
"cells": [
{
"cell_type": "code",
"source": [
"!pip install litellm"
],
"metadata": {
"id": "j6yJsCGeaq8G"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "u129iWNPaf72"
},
"outputs": [],
"source": [
"import litellm\n",
"from litellm import embedding, completion\n",
"\n",
"model_fallback_list = [\"claude-instant-1\", \"gpt-3.5-turbo\", \"chatgpt-test\"]\n",
"\n",
"user_message = \"Hello, how are you?\"\n",
"messages = [{ \"content\": user_message,\"role\": \"user\"}]\n",
"\n",
"for model in model_fallback_list:\n",
" try:\n",
" response = completion(model=model, messages=messages)\n",
" except Exception as e:\n",
" print(f\"error occurred: {traceback.format_exc()}\")"
]
}
]
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "u129iWNPaf72"
},
"outputs": [],
"source": [
"from litellm import completion\n",
"\n",
"model_fallback_list = [\"claude-instant-1\", \"gpt-3.5-turbo\", \"chatgpt-test\"]\n",
"\n",
"user_message = \"Hello, how are you?\"\n",
"messages = [{ \"content\": user_message,\"role\": \"user\"}]\n",
"\n",
"for model in model_fallback_list:\n",
" try:\n",
" response = completion(model=model, messages=messages)\n",
" except Exception:\n",
" print(f\"error occurred: {traceback.format_exc()}\")"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

25
cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py Normal file
View file

@ -0,0 +1,25 @@
from openai import OpenAI
client = OpenAI(
base_url="http://0.0.0.0:4000",
api_key="sk-1234",
)
BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
# Upload file
batch_input_file = client.files.create(
file=open("./bedrock_batch_completions.jsonl", "rb"),
purpose="batch",
extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
)
print(batch_input_file)
# Create batch
batch = client.batches.create(
input_file_id=batch_input_file.id,
endpoint="/v1/chat/completions",
completion_window="24h",
metadata={"description": "Test batch job"},
)
print(batch)

128
cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl Normal file
View file

@ -0,0 +1,128 @@
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}

279
cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md Normal file
View file

@ -0,0 +1,279 @@
# Braintrust Prompt Wrapper for LiteLLM
This directory contains a wrapper server that enables LiteLLM to use prompts from [Braintrust](https://www.braintrust.dev/) through the generic prompt management API.
## Architecture
```
┌─────────────┐ ┌──────────────────────┐ ┌─────────────┐
│ LiteLLM │ ──────> │ Wrapper Server │ ──────> │ Braintrust │
│ Client │ │ (This Server) │ │ API │
└─────────────┘ └──────────────────────┘ └─────────────┘
Uses generic Transforms Stores actual
prompt manager Braintrust format prompt templates
to LiteLLM format
```
## Components
### 1. Generic Prompt Manager (`litellm/integrations/generic_prompt_management/`)
A generic client that can work with any API implementing the `/beta/litellm_prompt_management` endpoint.
**Expected API Response Format:**
```json
{
"prompt_id": "string",
"prompt_template": [
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Hello {name}"}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 100
}
}
```
### 2. Braintrust Wrapper Server (`braintrust_prompt_wrapper_server.py`)
A FastAPI server that:
- Implements the `/beta/litellm_prompt_management` endpoint
- Fetches prompts from Braintrust API
- Transforms Braintrust response format to LiteLLM format
## Setup
### Install Dependencies
```bash
pip install fastapi uvicorn httpx litellm
```
### Set Environment Variables
```bash
export BRAINTRUST_API_KEY="your-braintrust-api-key"
```
## Usage
### Step 1: Start the Wrapper Server
```bash
python braintrust_prompt_wrapper_server.py
```
The server will start on `http://localhost:8080` by default.
You can customize the port and host:
```bash
export PORT=8000
export HOST=0.0.0.0
python braintrust_prompt_wrapper_server.py
```
### Step 2: Use with LiteLLM
```python
import litellm
from litellm.integrations.generic_prompt_management import GenericPromptManager
# Configure the generic prompt manager to use your wrapper server
generic_config = {
"api_base": "http://localhost:8080",
"api_key": "your-braintrust-api-key", # Will be passed to Braintrust
"timeout": 30,
}
# Create the prompt manager
prompt_manager = GenericPromptManager(**generic_config)
# Use with completion
response = litellm.completion(
model="generic_prompt/gpt-4",
prompt_id="your-braintrust-prompt-id",
prompt_variables={"name": "World"}, # Variables to substitute
messages=[{"role": "user", "content": "Additional message"}]
)
print(response)
```
### Step 3: Direct API Testing
You can also test the wrapper API directly:
```bash
# Test with curl
curl -H "Authorization: Bearer YOUR_BRAINTRUST_TOKEN" \
"http://localhost:8080/beta/litellm_prompt_management?prompt_id=YOUR_PROMPT_ID"
# Health check
curl http://localhost:8080/health
# Service info
curl http://localhost:8080/
```
## API Documentation
Once the server is running, visit:
- Swagger UI: `http://localhost:8080/docs`
- ReDoc: `http://localhost:8080/redoc`
## Braintrust Format Transformation
The wrapper automatically transforms Braintrust's response format:
**Braintrust API Response:**
```json
{
"id": "prompt-123",
"prompt_data": {
"prompt": {
"type": "chat",
"messages": [
{
"role": "system",
"content": "You are a helpful assistant"
}
]
},
"options": {
"model": "gpt-4",
"params": {
"temperature": 0.7,
"max_tokens": 100
}
}
}
}
```
**Transformed to LiteLLM Format:**
```json
{
"prompt_id": "prompt-123",
"prompt_template": [
{
"role": "system",
"content": "You are a helpful assistant"
}
],
"prompt_template_model": "gpt-4",
"prompt_template_optional_params": {
"temperature": 0.7,
"max_tokens": 100
}
}
```
## Supported Parameters
The wrapper automatically maps these Braintrust parameters to LiteLLM:
- `temperature`
- `max_tokens` / `max_completion_tokens`
- `top_p`
- `frequency_penalty`
- `presence_penalty`
- `n`
- `stop`
- `response_format`
- `tool_choice`
- `function_call`
- `tools`
## Variable Substitution
The generic prompt manager supports simple variable substitution:
```python
# In your Braintrust prompt:
# "Hello {name}, welcome to {place}!"
# In your code:
prompt_variables = {
"name": "Alice",
"place": "Wonderland"
}
# Result:
# "Hello Alice, welcome to Wonderland!"
```
Supports both `{variable}` and `{{variable}}` syntax.
## Error Handling
The wrapper provides detailed error messages:
- **401**: Missing or invalid Braintrust API token
- **404**: Prompt not found in Braintrust
- **502**: Failed to connect to Braintrust API
- **500**: Error transforming response
## Production Deployment
For production use:
1. **Use HTTPS**: Deploy behind a reverse proxy with SSL
2. **Authentication**: Add authentication to the wrapper endpoint if needed
3. **Rate Limiting**: Implement rate limiting to prevent abuse
4. **Caching**: Consider caching prompt responses
5. **Monitoring**: Add logging and monitoring
Example with Docker:
```dockerfile
FROM python:3.11-slim
WORKDIR /app
RUN pip install fastapi uvicorn httpx
COPY braintrust_prompt_wrapper_server.py .
ENV PORT=8080
ENV HOST=0.0.0.0
EXPOSE 8080
CMD ["python", "braintrust_prompt_wrapper_server.py"]
```
## Extending to Other Providers
This pattern can be used with any prompt management provider:
1. Create a wrapper server that implements `/beta/litellm_prompt_management`
2. Transform the provider's response to LiteLLM format
3. Use the generic prompt manager to connect
Example providers:
- Langsmith
- PromptLayer
- Humanloop
- Custom internal systems
## Troubleshooting
### "No Braintrust API token provided"
- Set `BRAINTRUST_API_KEY` environment variable
- Or pass token in `Authorization: Bearer TOKEN` header
### "Failed to connect to Braintrust API"
- Check your internet connection
- Verify Braintrust API is accessible
- Check firewall settings
### "Prompt not found"
- Verify the prompt ID exists in Braintrust
- Check that your API token has access to the prompt
## License
This wrapper is part of the LiteLLM project and follows the same license.

Some files were not shown because too many files have changed in this diff Show more