scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 87

Merge branch 'BerriAI:main' into main

Browse source
This commit is contained in:
Eric84626 2025-12-20 09:42:41 +08:00 • committed by GitHub
commit 96f7529f11
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
567 changed files with 43735 additions and 3666 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
.circleci/config.yml
View file

@ -657,7 +657,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2108,7 +2108,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2250,7 +2250,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2390,7 +2390,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2551,7 +2551,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2664,7 +2664,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2800,7 +2800,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -3032,7 +3032,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -3549,7 +3549,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14

84
.gitguardian.yaml Normal file
View file

@ -0,0 +1,84 @@
version: 2
secret:
# Exclude files and paths by globbing
ignored_paths:
- "**/*.whl"
- "**/*.pyc"
- "**/__pycache__/**"
- "**/node_modules/**"
- "**/dist/**"
- "**/build/**"
- "**/.git/**"
- "**/venv/**"
- "**/.venv/**"
# Large data/metadata files that don't need scanning
- "**/model_prices_and_context_window*.json"
- "**/*_metadata/*.txt"
- "**/tokenizers/*.json"
- "**/tokenizers/*"
- "miniconda.sh"
# Build outputs and static assets
- "litellm/proxy/_experimental/out/**"
- "ui/litellm-dashboard/public/**"
- "**/swagger/*.js"
- "**/*.woff"
- "**/*.woff2"
- "**/*.avif"
- "**/*.webp"
# Test data files
- "**/tests/**/data_map.txt"
- "tests/**/*.txt"
# Documentation and other non-code files
- "docs/**"
- "**/*.md"
- "**/*.lock"
- "poetry.lock"
- "package-lock.json"
# Ignore security incidents with the SHA256 of the occurrence (false positives)
ignored_matches:
# === Current detected false positives (SHA-based) ===
# gcs_pub_sub_body - folder name, not a password
- name: GCS pub/sub test folder name
match: 75f377c456eede69e5f6e47399ccee6016a2a93cc5dd11db09cc5b1359ae569a
# os.environ/APORIA_API_KEY_1 - environment variable reference
- name: Environment variable reference APORIA_API_KEY_1
match: e2ddeb8b88eca97a402559a2be2117764e11c074d86159ef9ad2375dea188094
# os.environ/APORIA_API_KEY_2 - environment variable reference
- name: Environment variable reference APORIA_API_KEY_2
match: 09aa39a29e050b86603aa55138af1ff08fb86a4582aa965c1bd0672e1575e052
# oidc/circleci_v2/ - test authentication path, not a secret
- name: OIDC CircleCI test path
match: feb3475e1f89a65b7b7815ac4ec597e18a9ec1847742ad445c36ca617b536e15
# text-davinci-003 - OpenAI model identifier, not a secret
- name: OpenAI model identifier text-davinci-003
match: c489000cf6c7600cee0eefb80ad0965f82921cfb47ece880930eb7e7635cf1f1
# === Preventive patterns for test keys (pattern-based) ===
# Test API keys (124 instances across 45 files)
- name: Test API keys with sk-test prefix
match: sk-test-
# Mock API keys
- name: Mock API keys with sk-mock prefix
match: sk-mock-
# Fake API keys
- name: Fake API keys with sk-fake prefix
match: sk-fake-
# Generic test API key patterns
- name: Test API key patterns
match: test-api-key

12
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View file

@ -23,13 +23,15 @@ body:
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
render: shell
- type: dropdown
id: ml-ops-team
id: component
attributes:
label: Are you a ML Ops Team?
description: This helps us prioritize your requests correctly
label: What part of LiteLLM is this about?
options:
- "No"
- "Yes"
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
- "Docs"
- "Other"
validations:
required: true
- type: input

12
.github/ISSUE_TEMPLATE/feature_request.yml vendored
View file

@ -22,6 +22,18 @@ body:
description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too.
validations:
required: true
- type: dropdown
id: component
attributes:
label: What part of LiteLLM is this about?
options:
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
- "Docs"
- "Other"
validations:
required: true
- type: dropdown
id: hiring-interest
attributes:

23
.github/pull_request_template.md vendored
View file

@ -1,7 +1,3 @@
## Title
<!-- e.g. "Implement user authentication feature" -->
## Relevant issues
<!-- e.g. "Fixes #000" -->
@ -11,10 +7,25 @@
**Please complete all items before asking a LiteLLM maintainer to review your PR**
- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] I have added a screenshot of my new test passing locally
- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
## CI (LiteLLM team)
> **CI status guideline:**
>
> - 50-55 passing tests: main is stable with minor issues.
> - 45-49 passing tests: acceptable but needs attention
> - <= 40 passing tests: unstable; be careful with your merges and assess the risk.
- [ ] **Branch creation CI run**
Link:
- [ ] **CI run for the last commit**
Link:
- [ ] **Merge / cherry-pick CI run**
Links:
## Type
@ -29,5 +40,3 @@
✅ Test
## Changes

43
.github/workflows/create_daily_staging_branch.yml vendored Normal file
View file

@ -0,0 +1,43 @@
name: Create Daily Staging Branch
on:
schedule:
- cron: '0 0 * * *' # Runs daily at midnight UTC
workflow_dispatch: # Allow manual trigger
jobs:
create-staging-branch:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Create daily staging branch
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Configure Git user
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
# Generate branch name with MM_DD_YYYY format
BRANCH_NAME="litellm_staging_$(date +'%m_%d_%Y')"
echo "Creating branch: $BRANCH_NAME"
# Fetch all branches
git fetch --all
# Check if the branch already exists
if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
echo "Branch $BRANCH_NAME already exists. Skipping creation."
else
echo "Creating new branch: $BRANCH_NAME"
# Create the new branch from main
git checkout -b $BRANCH_NAME origin/main
# Push the new branch
git push origin $BRANCH_NAME
echo "Successfully created and pushed branch: $BRANCH_NAME"
fi

2
.github/workflows/issue-keyword-labeler.yml vendored
View file

@ -19,7 +19,7 @@ jobs:
id: scan
env:
PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic
KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic,gemini,cohere,mistral,groq,ollama,deepseek
run: python3 .github/scripts/scan_keywords.py
- name: Ensure label exists

144
.github/workflows/label-component.yml vendored Normal file
View file

@ -0,0 +1,144 @@
name: Label Component Issues
on:
issues:
types:
- opened
jobs:
add-component-label:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Add SDK label
if: contains(github.event.issue.body, 'SDK (litellm Python package)')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'sdk';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: '0E7C86',
description: 'Issues related to the litellm Python SDK'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
- name: Add Proxy label
if: contains(github.event.issue.body, 'Proxy')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'proxy';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: '5319E7',
description: 'Issues related to the LiteLLM Proxy'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
- name: Add UI Dashboard label
if: contains(github.event.issue.body, 'UI Dashboard')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'ui-dashboard';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'D876E3',
description: 'Issues related to the LiteLLM UI Dashboard'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
- name: Add Docs label
if: contains(github.event.issue.body, 'Docs')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'docs';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'FBCA04',
description: 'Issues related to LiteLLM documentation'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});

17
.github/workflows/label-mlops.yml vendored
View file

@ -1,17 +0,0 @@
name: Label ML Ops Team Issues
on:
issues:
types:
- opened
jobs:
add-mlops-label:
runs-on: ubuntu-latest
steps:
- name: Check if ML Ops Team is selected
uses: actions-ecosystem/action-add-labels@v1
if: contains(github.event.issue.body, '### Are you a ML Ops Team?') && contains(github.event.issue.body, 'Yes')
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
labels: "mlops user request"

2
.github/workflows/locustfile.py vendored
View file

@ -8,7 +8,7 @@ class MyUser(HttpUser):
def chat_completion(self):
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg",
"Authorization": "Bearer sk-test-load-test-key-123",
# Include any additional headers you may need for authentication, etc.
}

4
.github/workflows/publish-migrations.yml vendored
View file

@ -20,7 +20,7 @@ jobs:
env:
POSTGRES_DB: temp_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_PASSWORD: test-postgres
ports:
- 5432:5432
options: >-
@ -35,7 +35,7 @@ jobs:
env:
POSTGRES_DB: shadow_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_PASSWORD: test-postgres
ports:
- 5433:5432
options: >-

428
README.md
View file

@ -2,16 +2,16 @@
🚅 LiteLLM
</h1>
<p align="center">
<p align="center">Call 100+ LLMs in OpenAI format. [Bedrock, Azure, OpenAI, VertexAI, Anthropic, Groq, etc.]
</p>
<p align="center">
<a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
<a href="https://railway.app/template/HLP0Ub?referralCode=jch2ME">
<img src="https://railway.app/button.svg" alt="Deploy on Railway">
</a>
</p>
<p align="center">Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]
<br>
</p>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (LLM Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
<h4 align="center">
<a href="https://pypi.org/project/litellm/" target="_blank">
<img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
@ -30,27 +30,17 @@
</a>
</h4>
LiteLLM manages:
<img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
## Use LiteLLM for
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs) <br>
[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
<details open>
<summary><b>LLMs</b> - Call 100+ LLMs (Python SDK + AI Gateway)</summary>
🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
[**All Supported Endpoints**](https://docs.litellm.ai/docs/supported_endpoints) - `/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, `/rerank`, `/a2a`, `/messages` and more.
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
# Usage ([**Docs**](https://docs.litellm.ai/docs/))
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
</a>
### Python SDK
```shell
pip install litellm
@ -60,249 +50,214 @@ pip install litellm
from litellm import completion
import os
## set ENV variables
os.environ["OPENAI_API_KEY"] = "your-openai-key"
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
messages = [{ "content": "Hello, how are you?","role": "user"}]
# OpenAI
response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hello!"}])
# openai call
response = completion(model="openai/gpt-4o", messages=messages)
# anthropic call
response = completion(model="anthropic/claude-sonnet-4-20250514", messages=messages)
print(response)
# Anthropic
response = completion(model="anthropic/claude-sonnet-4-20250514", messages=[{"role": "user", "content": "Hello!"}])
```
### Response (OpenAI Format)
### AI Gateway (Proxy Server)
```json
{
"id": "chatcmpl-1214900a-6cdd-4148-b663-b5e2f642b4de",
"created": 1751494488,
"model": "claude-sonnet-4-20250514",
"object": "chat.completion",
"system_fingerprint": null,
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Hello! I'm doing well, thank you for asking. I'm here and ready to help with whatever you'd like to discuss or work on. How are you doing today?",
"role": "assistant",
"tool_calls": null,
"function_call": null
}
}
],
"usage": {
"completion_tokens": 39,
"prompt_tokens": 13,
"total_tokens": 52,
"completion_tokens_details": null,
"prompt_tokens_details": {
"audio_tokens": null,
"cached_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0
}
}
```
> **Note:** LiteLLM also supports the [Responses API](https://docs.litellm.ai/docs/response_api) (`litellm.responses()`)
Call any model supported by a provider, with `model=<provider_name>/<model_name>`. There might be provider-specific details here, so refer to [provider docs for more information](https://docs.litellm.ai/docs/providers)
## Async ([Docs](https://docs.litellm.ai/docs/completion/stream#async-completion))
```python
from litellm import acompletion
import asyncio
async def test_get_response():
user_message = "Hello, how are you?"
messages = [{"content": user_message, "role": "user"}]
response = await acompletion(model="openai/gpt-4o", messages=messages)
return response
response = asyncio.run(test_get_response())
print(response)
```
## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
LiteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
```python
from litellm import completion
messages = [{"content": "Hello, how are you?", "role": "user"}]
# gpt-4o
response = completion(model="openai/gpt-4o", messages=messages, stream=True)
for part in response:
print(part.choices[0].delta.content or "")
# claude sonnet 4
response = completion('anthropic/claude-sonnet-4-20250514', messages, stream=True)
for part in response:
print(part)
```
### Response chunk (OpenAI Format)
```json
{
"id": "chatcmpl-fe575c37-5004-4926-ae5e-bfbc31f356ca",
"created": 1751494808,
"model": "claude-sonnet-4-20250514",
"object": "chat.completion.chunk",
"system_fingerprint": null,
"choices": [
{
"finish_reason": null,
"index": 0,
"delta": {
"provider_specific_fields": null,
"content": "Hello",
"role": "assistant",
"function_call": null,
"tool_calls": null,
"audio": null
},
"logprobs": null
}
],
"provider_specific_fields": null,
"stream_options": null,
"citations": null
}
```
## Logging Observability ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack
```python
from litellm import completion
## set env variables for logging tools (when using MLflow, no API key set up is required)
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
os.environ["HELICONE_API_KEY"] = "your-helicone-auth-key"
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
os.environ["LANGFUSE_SECRET_KEY"] = ""
os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
os.environ["OPENAI_API_KEY"] = "your-openai-key"
# set callbacks
litellm.success_callback = ["lunary", "mlflow", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
#openai call
response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
```
# LiteLLM Proxy Server (LLM Gateway) - ([Docs](https://docs.litellm.ai/docs/simple_proxy))
Track spend + Load Balance across multiple projects
[Hosted Proxy](https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy)
The proxy provides:
1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth)
2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class)
3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend)
4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits)
## 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/)
## Quick Start Proxy - CLI
[**Getting Started - E2E Tutorial**](https://docs.litellm.ai/docs/proxy/docker_quick_start) - Setup virtual keys, make your first request
```shell
pip install 'litellm[proxy]'
litellm --model gpt-4o
```
### Step 1: Start litellm proxy
```shell
$ litellm --model huggingface/bigcode/starcoder
#INFO: Proxy running on http://0.0.0.0:4000
```
### Step 2: Make ChatCompletions Request to Proxy
> [!IMPORTANT]
> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
```python
import openai # openai v1.0.0+
client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
])
import openai
print(response)
client = openai.OpenAI(api_key="anything", base_url="http://0.0.0.0:4000")
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello!"}]
)
```
## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys))
[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
Connect the proxy with a Postgres DB to create proxy keys
</details>
<details>
<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
[**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
### Python SDK - A2A Protocol
```python
from litellm.a2a_protocol import A2AClient
from a2a.types import SendMessageRequest, MessageSendParams
from uuid import uuid4
client = A2AClient(base_url="http://localhost:10001")
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello!"}],
"messageId": uuid4().hex,
}
)
)
response = await client.send_message(request)
```
### AI Gateway (Proxy Server)
**Step 1.** [Add your Agent to the AI Gateway](https://docs.litellm.ai/docs/a2a#adding-your-agent)
**Step 2.** Call Agent via A2A SDK
```python
from a2a.client import A2ACardResolver, A2AClient
from a2a.types import MessageSendParams, SendMessageRequest
from uuid import uuid4
import httpx
base_url = "http://localhost:4000/a2a/my-agent" # LiteLLM proxy + agent name
headers = {"Authorization": "Bearer sk-1234"} # LiteLLM Virtual Key
async with httpx.AsyncClient(headers=headers) as httpx_client:
resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
agent_card = await resolver.get_agent_card()
client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello!"}],
"messageId": uuid4().hex,
}
)
)
response = await client.send_message(request)
```
[**Docs: A2A Agent Gateway**](https://docs.litellm.ai/docs/a2a)
</details>
<details>
<summary><b>MCP Tools</b> - Connect MCP servers to any LLM (Python SDK + AI Gateway)</summary>
### Python SDK - MCP Bridge
```python
from mcp import ClientSession, StdioServerParameters
from mcp.client.stdio import stdio_client
from litellm import experimental_mcp_client
import litellm
server_params = StdioServerParameters(command="python", args=["mcp_server.py"])
async with stdio_client(server_params) as (read, write):
async with ClientSession(read, write) as session:
await session.initialize()
# Load MCP tools in OpenAI format
tools = await experimental_mcp_client.load_mcp_tools(session=session, format="openai")
# Use with any LiteLLM model
response = await litellm.acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": "What's 3 + 5?"}],
tools=tools
)
```
### AI Gateway - MCP Gateway
**Step 1.** [Add your MCP Server to the AI Gateway](https://docs.litellm.ai/docs/mcp#adding-your-mcp)
**Step 2.** Call MCP tools via `/chat/completions`
```bash
# Get the code
git clone https://github.com/BerriAI/litellm
# Go to folder
cd litellm
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# Add the litellm salt key - you cannot change this after adding a model
# It is used to encrypt / decrypt your LLM API Key credentials
# We recommend - https://1password.com/password-generator/
# password generator to get a random hash for litellm salt key
echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
# Start
docker compose up
curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Summarize the latest open PR"}],
"tools": [{
"type": "mcp",
"server_url": "litellm_proxy/mcp/github",
"server_label": "github_mcp",
"require_approval": "never"
}]
}'
```
### Use with Cursor IDE
UI on `/ui` on your proxy server
![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033)
Set budgets and rate limits across multiple projects
`POST /key/generate`
### Request
```shell
curl 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data-raw '{"models": ["gpt-3.5-turbo", "gpt-4", "claude-2"], "duration": "20m","metadata": {"user": "ishaan@berri.ai", "team": "core-infra"}}'
```
### Expected Response
```shell
```json
{
"key": "sk-kdEXbIqZRwEeEiHwdg7sFA", # Bearer token
"expires": "2023-11-19T01:38:25.838000+00:00" # datetime object
"mcpServers": {
"LiteLLM": {
"url": "http://localhost:4000/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
}
}
}
```
[**Docs: MCP Gateway**](https://docs.litellm.ai/docs/mcp)
</details>
---
## How to use LiteLLM
You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs:
<table style={{width: '100%', tableLayout: 'fixed'}}>
<thead>
<tr>
<th style={{width: '14%'}}></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
</tr>
</thead>
<tbody>
<tr>
<td style={{width: '14%'}}><strong>Use Case</strong></td>
<td style={{width: '43%'}}>Central service (LLM Gateway) to access multiple LLMs</td>
<td style={{width: '43%'}}>Use LiteLLM directly in your Python code</td>
</tr>
<tr>
<td style={{width: '14%'}}><strong>Who Uses It?</strong></td>
<td style={{width: '43%'}}>Gen AI Enablement / ML Platform Teams</td>
<td style={{width: '43%'}}>Developers building LLM projects</td>
</tr>
<tr>
<td style={{width: '14%'}}><strong>Key Features</strong></td>
<td style={{width: '43%'}}>Centralized API gateway with authentication and authorization, multi-tenant cost tracking and spend management per project/user, per-project customization (logging, guardrails, caching), virtual keys for secure access control, admin dashboard UI for monitoring and management</td>
<td style={{width: '43%'}}>Direct Python library integration in your codebase, Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - <a href="https://docs.litellm.ai/docs/routing">Router</a>, application-level load balancing and cost tracking, exception handling with OpenAI-compatible errors, observability callbacks (Lunary, MLflow, Langfuse, etc.)</td>
</tr>
</tbody>
</table>
LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://docs.litellm.ai/docs/simple_proxy) <br>
[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
**Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
@ -311,6 +266,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [Aleph Alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | | | | | | | |
| [Amazon Nova](https://docs.litellm.ai/docs/providers/amazon_nova) | ✅ | ✅ | ✅ | | | | | | | |
| [Anthropic (`anthropic`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
| [Anthropic Text (`anthropic_text`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
| [Anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | | | | | | | |

40
ci_cd/TEST_KEY_PATTERNS.md Normal file
View file

@ -0,0 +1,40 @@
# Test Key Patterns Standard
Standard patterns for test/mock keys and credentials in the LiteLLM codebase to avoid triggering secret detection.
## How GitGuardian Works
GitGuardian uses **machine learning and entropy analysis**, not just pattern matching:
- **Low entropy** values (like `sk-1234`, `postgres`) are automatically ignored
- **High entropy** values (realistic-looking secrets) trigger detection
- **Context-aware** detection understands code syntax like `os.environ["KEY"]`
## Recommended Test Key Patterns
### Option 1: Low Entropy Values (Simplest)
These won't trigger GitGuardian's ML detector:
```python
api_key = "sk-1234"
api_key = "sk-12345"
database_password = "postgres"
token = "test123"
```
### Option 2: High Entropy with Test Prefixes
If you need realistic-looking test keys with high entropy, use these prefixes:
```python
api_key = "sk-test-abc123def456ghi789..." # OpenAI-style test key
api_key = "sk-mock-1234567890abcdef1234..." # Mock key
api_key = "sk-fake-xyz789uvw456rst123..." # Fake key
token = "test-api-key-with-high-entropy"
```
## Configured Ignore Patterns
These patterns are in `.gitguardian.yaml` for high-entropy test keys:
- `sk-test-*` - OpenAI-style test keys
- `sk-mock-*` - Mock API keys
- `sk-fake-*` - Fake API keys
- `test-api-key` - Generic test tokens

53
ci_cd/security_scans.sh
View file

@ -26,6 +26,56 @@ install_grype() {
echo "Grype installed successfully"
}
# Function to install ggshield
install_ggshield() {
echo "Installing ggshield..."
pip3 install --upgrade pip
pip3 install ggshield
echo "ggshield installed successfully"
}
# Function to run secret detection scans
run_secret_detection() {
echo "Running secret detection scans..."
if ! command -v ggshield &> /dev/null; then
install_ggshield
fi
# Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
if [ -z "$GITGUARDIAN_API_KEY" ]; then
echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
echo "ggshield requires a GitGuardian API key to scan for secrets."
echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
exit 1
fi
echo "Scanning codebase for secrets..."
echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
echo "ggshield will automatically handle rate limits and retry as needed."
echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
# Use --recursive for directory scanning and auto-confirm if prompted
# .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
# GITGUARDIAN_API_KEY environment variable will be used for authentication
echo y | ggshield secret scan path . --recursive || {
echo ""
echo "=========================================="
echo "ERROR: Secret Detection Failed"
echo "=========================================="
echo "ggshield has detected secrets in the codebase."
echo "Please review discovered secrets above, revoke any actively used secrets"
echo "from underlying systems and make changes to inject secrets dynamically at runtime."
echo ""
echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
echo "=========================================="
echo ""
exit 1
}
echo "Secret detection scans completed successfully"
}
# Function to run Trivy scans
run_trivy_scans() {
echo "Running Trivy scans..."
@ -158,6 +208,9 @@ main() {
install_trivy
install_grype
echo "Running secret detection scans..."
run_secret_detection
echo "Running filesystem vulnerability scans..."
run_trivy_scans

2
cookbook/LiteLLM_PromptLayer.ipynb vendored
View file

@ -39,7 +39,7 @@
"import os\n",
"os.environ['OPENAI_API_KEY'] = \"\"\n",
"os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
"os.environ['PROMPTLAYER_API_KEY'] = \"pl_4ea2bb00a4dca1b8a70cebf2e9e11564\"\n",
"os.environ['PROMPTLAYER_API_KEY'] = \"test-promptlayer-key-123\"\n",
"\n",
"# Set Promptlayer as a success callback\n",
"litellm.success_callback =['promptlayer']\n",

278
cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb vendored
View file

@ -1,21 +1,10 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "kccfk0mHZ4Ad"
},
"source": [
"# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n",
"\n",
@ -32,29 +21,26 @@
"To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
"\n",
"To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n"
],
"metadata": {
"id": "kccfk0mHZ4Ad"
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "nmSClzCPaGH6"
},
"source": [
"## /chat/completion\n",
"\n"
],
"metadata": {
"id": "nmSClzCPaGH6"
}
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI Python SDK"
],
"metadata": {
"id": "_vqcjwOVaKpO"
}
},
"source": [
"### OpenAI Python SDK"
]
},
{
"cell_type": "code",
@ -94,15 +80,20 @@
},
{
"cell_type": "markdown",
"source": [
"## Function Calling"
],
"metadata": {
"id": "AqkyKk9Scxgj"
}
},
"source": [
"## Function Calling"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "wDg10VqLczE1"
},
"outputs": [],
"source": [
"from openai import OpenAI\n",
"client = OpenAI(\n",
@ -139,24 +130,24 @@
")\n",
"\n",
"print(completion)\n"
],
"metadata": {
"id": "wDg10VqLczE1"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Azure OpenAI Python SDK"
],
"metadata": {
"id": "YYoxLloSaNWW"
}
},
"source": [
"### Azure OpenAI Python SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "yA1XcgowaSRy"
},
"outputs": [],
"source": [
"import openai\n",
"client = openai.AzureOpenAI(\n",
@ -184,24 +175,24 @@
")\n",
"\n",
"print(response)"
],
"metadata": {
"id": "yA1XcgowaSRy"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain Python"
],
"metadata": {
"id": "yl9qhDvnaTpL"
}
},
"source": [
"### Langchain Python"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "5MUZgSquaW5t"
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.prompts.chat import (\n",
@ -239,24 +230,22 @@
"response = chat(messages)\n",
"\n",
"print(response)"
],
"metadata": {
"id": "5MUZgSquaW5t"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Curl"
],
"metadata": {
"id": "B9eMgnULbRaz"
}
},
"source": [
"### Curl"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "VWCCk5PFcmhS"
},
"source": [
"\n",
"\n",
@ -280,22 +269,24 @@
"}'\n",
"```\n",
"\n"
],
"metadata": {
"id": "VWCCk5PFcmhS"
}
]
},
{
"cell_type": "markdown",
"source": [
"### LlamaIndex"
],
"metadata": {
"id": "drBAm2e1b6xe"
}
},
"source": [
"### LlamaIndex"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "d0bZcv8fb9mL"
},
"outputs": [],
"source": [
"import os, dotenv\n",
"\n",
@ -326,24 +317,24 @@
"query_engine = index.as_query_engine()\n",
"response = query_engine.query(\"What did the author do growing up?\")\n",
"print(response)\n"
],
"metadata": {
"id": "d0bZcv8fb9mL"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain JS"
],
"metadata": {
"id": "xypvNdHnb-Yy"
}
},
"source": [
"### Langchain JS"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "R55mK2vCcBN2"
},
"outputs": [],
"source": [
"import { ChatOpenAI } from \"@langchain/openai\";\n",
"\n",
@ -359,24 +350,24 @@
"const message = await model.invoke(\"Hi there!\");\n",
"\n",
"console.log(message);\n"
],
"metadata": {
"id": "R55mK2vCcBN2"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI JS"
],
"metadata": {
"id": "nC4bLifCcCiW"
}
},
"source": [
"### OpenAI JS"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "MICH8kIMcFpg"
},
"outputs": [],
"source": [
"const { OpenAI } = require('openai');\n",
"\n",
@ -398,24 +389,24 @@
"}\n",
"\n",
"main();\n"
],
"metadata": {
"id": "MICH8kIMcFpg"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Anthropic SDK"
],
"metadata": {
"id": "D1Q07pEAcGTb"
}
},
"source": [
"### Anthropic SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "qBjFcAvgcI3t"
},
"outputs": [],
"source": [
"import os\n",
"\n",
@ -423,7 +414,7 @@
"\n",
"client = Anthropic(\n",
" base_url=\"http://localhost:4000\", # proxy endpoint\n",
" api_key=\"sk-s4xN1IiLTCytwtZFJaYQrA\", # litellm proxy virtual key\n",
" api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n",
")\n",
"\n",
"message = client.messages.create(\n",
@ -437,33 +428,33 @@
" model=\"claude-3-opus-20240229\",\n",
")\n",
"print(message.content)"
],
"metadata": {
"id": "qBjFcAvgcI3t"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"## /embeddings"
],
"metadata": {
"id": "dFAR4AJGcONI"
}
},
"source": [
"## /embeddings"
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI Python SDK"
],
"metadata": {
"id": "lgNoM281cRzR"
}
},
"source": [
"### OpenAI Python SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "NY3DJhPfcQhA"
},
"outputs": [],
"source": [
"import openai\n",
"from openai import OpenAI\n",
@ -478,24 +469,24 @@
")\n",
"\n",
"print(response)\n"
],
"metadata": {
"id": "NY3DJhPfcQhA"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain Embeddings"
],
"metadata": {
"id": "hmbg-DW6cUZs"
}
},
"source": [
"### Langchain Embeddings"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "lX2S8Nl1cWVP"
},
"outputs": [],
"source": [
"from langchain.embeddings import OpenAIEmbeddings\n",
"\n",
@ -526,24 +517,22 @@
"\n",
"print(f\"TITAN EMBEDDINGS\")\n",
"print(query_result[:5])"
],
"metadata": {
"id": "lX2S8Nl1cWVP"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Curl Request"
],
"metadata": {
"id": "oqGbWBCQcYfd"
}
},
"source": [
"### Curl Request"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7rkIMV9LcdwQ"
},
"source": [
"\n",
"\n",
@ -556,10 +545,21 @@
" }'\n",
"```\n",
"\n"
],
"metadata": {
"id": "7rkIMV9LcdwQ"
}
]
}
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

2
deploy/charts/litellm-helm/README.md
View file

@ -29,7 +29,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A |
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` |
| `image.repository` | LiteLLM Proxy image repository | `docker.litellm.ai/berriai/litellm` |
| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` |
| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` |
| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` |

46
docker-compose.hardened.yml Normal file
View file

@ -0,0 +1,46 @@
services:
# Hardened stack: for testing the proxy under non-root, read-only, proxy-enforced constraints.
# Keep this file focused on hardening/QA scenarios; leave the main docker-compose.yml for default dev usage.
litellm:
build:
context: .
dockerfile: docker/Dockerfile.non_root
target: runtime
args:
PROXY_EXTRAS_SOURCE: "local"
depends_on:
- squid
user: "101:101"
group_add:
- "2345"
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
tmpfs:
- /app/cache:rw,noexec,nosuid,nodev,size=128m,uid=101,gid=101,mode=1777
- /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777
volumes:
- ./proxy_server_config.yaml:/app/config.yaml:ro
environment:
LITELLM_NON_ROOT: "true"
PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries"
XDG_CACHE_HOME: "/app/cache"
LITELLM_MIGRATION_DIR: "/app/migrations"
HTTP_PROXY: "http://squid:3128"
HTTPS_PROXY: "http://squid:3128"
NO_PROXY: "localhost,127.0.0.1,db"
command:
- "--port"
- "4000"
- "--config"
- "/app/config.yaml"
squid:
image: sameersbn/squid:3.5.27-2
restart: unless-stopped
ports:
- "3128:3128"
tmpfs:
- /var/spool/squid:rw,noexec,nosuid,nodev,size=64m
- /var/log/squid:rw,noexec,nosuid,nodev,size=16m

2
docker-compose.yml
View file

@ -4,7 +4,7 @@ services:
context: .
args:
target: runtime
image: ghcr.io/berriai/litellm:main-stable
image: docker.litellm.ai/berriai/litellm:main-stable
#########################################
## Uncomment these lines to start proxy with a config.yaml file ##
# volumes:

4
docker/Dockerfile.alpine
View file

@ -34,8 +34,8 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE AS runtime
# Update dependencies and clean up
RUN apk upgrade --no-cache
# Update dependencies and clean up, install libsndfile for audio processing
RUN apk upgrade --no-cache && apk add --no-cache libsndfile
WORKDIR /app

255
docker/Dockerfile.non_root
View file

@ -1,154 +1,183 @@
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
ARG PROXY_EXTRAS_SOURCE=published
# -----------------
# Builder Stage
# -----------------
FROM $LITELLM_BUILD_IMAGE AS builder
ARG PROXY_EXTRAS_SOURCE
WORKDIR /app
# Install build dependencies including Node.js for UI build
USER root
# Install build dependencies with retry logic (includes node for UI build)
RUN for i in 1 2 3; do \
apk add --no-cache \
python3 \
py3-pip \
clang \
llvm \
lld \
gcc \
linux-headers \
build-base \
bash \
nodejs \
npm && break || sleep 5; \
done \
apk add --no-cache \
python3 \
py3-pip \
clang \
llvm \
lld \
gcc \
linux-headers \
build-base \
bash \
nodejs \
npm && break || sleep 5; \
done \
&& pip install --no-cache-dir --upgrade pip build
# Copy project files
# Cache Python dependencies
COPY requirements.txt .
RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \
&& pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.9.0"
# Copy source after dependency layers
COPY . .
# Set LITELLM_NON_ROOT flag for build time
# Set non-root flag for build time consistency
ENV LITELLM_NON_ROOT=true
# Build Admin UI
RUN mkdir -p /tmp/litellm_ui
# Build Admin UI using the upstream command order while keeping a single RUN layer
RUN mkdir -p /tmp/litellm_ui && \
npm install -g npm@latest && npm cache clean --force && \
cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
rm -f package-lock.json && \
npm install --legacy-peer-deps && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ && \
mkdir -p /tmp/litellm_assets && \
cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg && \
( cd /tmp/litellm_ui && \
for html_file in *.html; do \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
mkdir -p "$folder_name" && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
done ) && \
cd /app/ui/litellm-dashboard && rm -rf ./out
RUN npm install -g npm@latest && npm cache clean --force
RUN cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi
RUN cd /app/ui/litellm-dashboard && rm -f package-lock.json
RUN cd /app/ui/litellm-dashboard && npm install --legacy-peer-deps
RUN cd /app/ui/litellm-dashboard && npm run build
RUN cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/
RUN mkdir -p /tmp/litellm_assets && cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg
RUN cd /tmp/litellm_ui && \
for html_file in *.html; do \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
mkdir -p "$folder_name" && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
done
RUN cd /app/ui/litellm-dashboard && rm -rf ./out
# Build package and wheel dependencies
# Build litellm wheel and place it in wheels dir (replace any PyPI wheels)
RUN rm -rf dist/* && python -m build && \
pip install dist/*.whl && \
pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
rm -f /wheels/litellm-*.whl && \
cp dist/*.whl /wheels/
# Optionally build local litellm-proxy-extras wheel
RUN if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \
cd /app/litellm-proxy-extras && rm -rf dist && python -m build && \
cp dist/*.whl /wheels/; \
fi
# Pre-cache Prisma binaries in the builder stage
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
XDG_CACHE_HOME=/app/.cache \
PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
RUN pip install --no-cache-dir prisma==0.11.0 nodejs-bin==18.4.0a4 \
&& mkdir -p /app/.cache/npm
RUN NPM_CONFIG_CACHE=/app/.cache/npm \
python -c "import prisma.cli.prisma as p; p.ensure_cached()"
RUN prisma generate && \
prisma --version && \
prisma migrate diff --from-empty --to-schema-datamodel ./schema.prisma --script > /dev/null 2>&1 || true
# -----------------
# Runtime Stage
# -----------------
FROM $LITELLM_RUNTIME_IMAGE AS runtime
ARG PROXY_EXTRAS_SOURCE
WORKDIR /app
# Install runtime dependencies
USER root
RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done
# Copy only necessary artifacts from builder stage for runtime
COPY . .
# Install runtime dependencies with retry
RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done
# Copy artifacts from builder
COPY --from=builder /app/requirements.txt /app/requirements.txt
COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
COPY --from=builder /app/schema.prisma /app/schema.prisma
COPY --from=builder /app/dist/*.whl .
COPY --from=builder /app/schema.prisma /app/
COPY --from=builder /wheels/ /wheels/
COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui
COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets
COPY --from=builder /app/.cache /app/.cache
COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
COPY --from=builder \
/usr/lib/python3.13/site-packages/nodejs* \
/usr/lib/python3.13/site-packages/prisma* \
/usr/lib/python3.13/site-packages/tomlkit* \
/usr/lib/python3.13/site-packages/nodeenv* \
/usr/lib/python3.13/site-packages/
COPY --from=builder /usr/bin/prisma /usr/bin/prisma
# Install package from wheel and dependencies
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
&& rm -f *.whl \
&& rm -rf /wheels
# Final runtime environment configuration
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
HOME=/app \
LITELLM_NON_ROOT=true \
XDG_CACHE_HOME=/app/.cache
# Remove test files and keys from dependencies
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
# Install packages from wheels and optional extras without network
RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
pip install --no-index --find-links=/wheels/ /wheels/litellm-*-py3-none-any.whl && \
pip install --no-index --find-links=/wheels/ --no-deps semantic_router==0.1.11 && \
pip install --no-index --find-links=/wheels/ aurelio-sdk==0.0.19 && \
if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \
if ls /wheels/litellm_proxy_extras-*.whl >/dev/null 2>&1; then \
pip install --no-index --find-links=/wheels/ /wheels/litellm_proxy_extras-*.whl; \
else \
echo "litellm_proxy_extras wheel not found; skipping local install"; \
fi; \
fi
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Permissions, cleanup, and Prisma prep
RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
mkdir -p /nonexistent /.npm /tmp/litellm_assets /tmp/litellm_ui && \
chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \
pip uninstall jwt -y || true && \
pip uninstall PyJWT -y || true && \
pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \
rm -rf /wheels && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \
LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+rX $PRISMA_PATH && \
chmod -R g+rX /app/.cache && \
mkdir -p /tmp/.npm /nonexistent /.npm && \
prisma generate
# Ensure correct JWT library is used (pyjwt not jwt)
RUN pip uninstall jwt -y && \
pip uninstall PyJWT -y && \
pip install PyJWT==2.9.0 --no-cache-dir
# Set Prisma cache directories
ENV PRISMA_BINARY_CACHE_DIR=/nonexistent
ENV NPM_CONFIG_CACHE=/.npm
# Install prisma and make entrypoints executable
RUN pip install --no-cache-dir prisma && \
chmod +x docker/entrypoint.sh && \
chmod +x docker/prod_entrypoint.sh
# Create directories and set permissions for non-root user
RUN mkdir -p /nonexistent /.npm /tmp/litellm_assets && \
chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
# OpenShift compatibility
RUN PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true
# Switch to non-root user
# Switch to non-root user for runtime
USER nobody
# Set HOME for prisma generate to have a writable directory
ENV HOME=/app
# Set LITELLM_NON_ROOT flag for runtime
ENV LITELLM_NON_ROOT=true
RUN prisma generate
# Prisma runtime knobs for offline containers
ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
PRISMA_HIDE_UPDATE_MESSAGE=1 \
PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
NPM_CONFIG_CACHE=/app/.cache/npm \
NPM_CONFIG_PREFER_OFFLINE=true \
PRISMA_OFFLINE_MODE=true
EXPOSE 4000/tcp
ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]
CMD ["--port", "4000"]

24
docker/README.md
View file

@ -59,6 +59,30 @@ To stop the running containers, use the following command:
docker compose down
```
## Hardened / Offline Testing
To ensure changes are safe for non-root, read-only root filesystems and restricted egress, always validate with the hardened compose file:
```bash
docker compose -f docker-compose.yml -f docker-compose.hardened.yml build --no-cache
docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d
```
This setup:
- Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image.
- Runs the proxy as a non-root user with a read-only rootfs and only two writable tmpfs mounts:
- `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`)
- `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`)
- Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines.
You should also verify offline Prisma behaviour with:
```bash
docker run --rm --network none --entrypoint prisma ghcr.io/berriai/litellm:main-stable --version
```
This command should succeed (showing engine versions) even with `--network none`, confirming that Prisma binaries are available without network access.
## Troubleshooting
- **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.

2
docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md
View file

@ -6,7 +6,7 @@ authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/

2
docs/my-website/blog/gemini_3/index.md
View file

@ -6,7 +6,7 @@ authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/

254
docs/my-website/blog/gemini_3_flash/index.md Normal file
View file

@ -0,0 +1,254 @@
---
slug: gemini_3_flash
title: "DAY 0 Support: Gemini 3 Flash on LiteLLM"
date: 2025-12-17T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Gemini 3 Flash Day 0 Support
LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along with it.
:::note
If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
:::
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.80.8-stable.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.80.8.post1
```
</TabItem>
</Tabs>
## What's New
### 1. New Thinking Levels: `thinkingLevel` with MINIMAL & MEDIUM
Gemini 3 Flash introduces granular thinking control with `thinkingLevel` instead of `thinkingBudget`.
- **MINIMAL**: Ultra-lightweight thinking for fast responses
- **MEDIUM**: Balanced thinking for complex reasoning
- **HIGH**: Maximum reasoning depth
LiteLLM automatically maps the OpenAI `reasoning_effort` parameter to Gemini's `thinkingLevel`, so you can use familiar `reasoning_effort` values (`minimal`, `low`, `medium`, `high`) without changing your code!
### 2. Thought Signatures
Like `gemini-3-pro`, this model also includes thought signatures for tool calls. LiteLLM handles signature extraction and embedding internally. [Learn more about thought signatures](../gemini_3/index.md#thought-signatures).
**Edge Case Handling**: If thought signatures are missing in the request, LiteLLM adds a dummy signature ensuring the API call doesn't break
---
## Supported Endpoints
LiteLLM provides **full end-to-end support** for Gemini 3 Flash on:
- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint
All endpoints support:
- Streaming and non-streaming responses
- Function calling with thought signatures
- Multi-turn conversations
- All Gemini 3-specific features
- Converstion of provider specific thinking related param to thinkingLevel
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
**Basic Usage with MEDIUM thinking (NEW)**
```python
from litellm import completion
# No need to make any changes to your code as we map openai reasoning param to thinkingLevel
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "Solve this complex math problem: 25 * 4 + 10"}],
reasoning_effort="medium", # NEW: MEDIUM thinking level
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gemini-3-flash
litellm_params:
model: gemini/gemini-3-flash-preview
api_key: os.environ/GEMINI_API_KEY
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
```
**3. Call with MEDIUM thinking**
```bash
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-d '{
"model": "gemini-3-flash",
"messages": [{"role": "user", "content": "Complex reasoning task"}],
"reasoning_effort": "medium"
}'
``'
</TabItem>
</Tabs>
---
## All `reasoning_effort` Levels
<Tabs>
<TabItem value="minimal" label="MINIMAL">
**Ultra-fast, minimal reasoning**
```python
from litellm import completion
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "What's 2+2?"}],
reasoning_effort="minimal",
)
```
</TabItem>
<TabItem value="low" label="LOW">
**Simple instruction following**
```python
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "Write a haiku about coding"}],
reasoning_effort="low",
)
```
</TabItem>
<TabItem value="medium" label="MEDIUM (NEW)">
**Balanced reasoning for complex tasks** ✨
```python
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "Analyze this dataset and find patterns"}],
reasoning_effort="medium", # NEW!
)
```
</TabItem>
<TabItem value="high" label="HIGH">
**Maximum reasoning depth**
```python
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "Prove this mathematical theorem"}],
reasoning_effort="high",
)
```
</TabItem>
</Tabs>
---
## Key Features
✅ **Thinking Levels**: MINIMAL, LOW, MEDIUM, HIGH
✅ **Thought Signatures**: Track reasoning with unique identifiers
✅ **Seamless Integration**: Works with existing OpenAI-compatible client
✅ **Backward Compatible**: Gemini 2.5 models continue using `thinkingBudget`
---
## Installation
```bash
pip install litellm --upgrade
```
```python
import litellm
from litellm import completion
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "Your question here"}],
reasoning_effort="medium", # Use MEDIUM thinking
)
print(response)
```
:::note
If using this model via vertex_ai, keep the location as global as this is the only supported location as of now.
:::
## `reasoning_effort` Mapping for Gemini 3+
| reasoning_effort | thinking_level |
|------------------|----------------|
| `minimal` | `minimal` |
| `low` | `low` |
| `medium` | `medium` |
| `high` | `high` |
| `disable` | `minimal` |
| `none` | `minimal` |

25
docs/my-website/docs/a2a.md
View file

@ -16,10 +16,12 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque
| Feature | Supported |
|---------|-----------|
| Supported Agent Providers | A2A, Vertex AI Agent Engine, LangGraph, Azure AI Foundry, Bedrock AgentCore, Pydantic AI |
| Logging | ✅ |
| Load Balancing | ✅ |
| Streaming | ✅ |
:::tip
LiteLLM follows the [A2A (Agent-to-Agent) Protocol](https://github.com/google/A2A) for invoking agents.
@ -28,6 +30,8 @@ LiteLLM follows the [A2A (Agent-to-Agent) Protocol](https://github.com/google/A2
## Adding your Agent
### Add A2A Agents
You can add A2A-compatible agents through the LiteLLM Admin UI.
1. Navigate to the **Agents** tab
@ -41,6 +45,27 @@ You can add A2A-compatible agents through the LiteLLM Admin UI.
The URL should be the invocation URL for your A2A agent (e.g., `http://localhost:10001`).
### Add Azure AI Foundry Agents
Follow [this guide, to add your azure ai foundry agent to LiteLLM Agent Gateway](./providers/azure_ai_agents#litellm-a2a-gateway)
### Add Vertex AI Agent Engine
Follow [this guide, to add your Vertex AI Agent Engine to LiteLLM Agent Gateway](./providers/vertex_ai_agent_engine)
### Add Bedrock AgentCore Agents
Follow [this guide, to add your bedrock agentcore agent to LiteLLM Agent Gateway](./providers/bedrock_agentcore#litellm-a2a-gateway)
### Add LangGraph Agents
Follow [this guide, to add your langgraph agent to LiteLLM Agent Gateway](./providers/langgraph#litellm-a2a-gateway)
### Add Pydantic AI Agents
Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./providers/pydantic_ai_agent#litellm-a2a-gateway)
## Invoking your Agents
Use the [A2A Python SDK](https://pypi.org/project/a2a/) to invoke agents through LiteLLM.

3
docs/my-website/docs/batches.md
View file

@ -7,7 +7,7 @@ Covers Batches, Files
| Feature | Supported | Notes |
|-------|-------|-------|
| Supported Providers | OpenAI, Azure, Vertex, Bedrock | - |
| Supported Providers | OpenAI, Azure, Vertex, Bedrock, vLLM | - |
| ✨ Cost Tracking | ✅ | LiteLLM Enterprise only |
| Logging | ✅ | Works across all logging integrations |
@ -430,6 +430,7 @@ All batch and file endpoints support model-based routing:
### [OpenAI](#quick-start)
### [Vertex AI](./providers/vertex#batch-apis)
### [Bedrock](./providers/bedrock_batches)
### [vLLM](./providers/vllm_batches)
## How Cost Tracking for Batches API Works

54
docs/my-website/docs/benchmarks.md
View file

@ -60,6 +60,58 @@ Each machine deploying LiteLLM had the following specs:
- Database: PostgreSQL
- Redis: Not used
## Infrastructure Recommendations
Recommended specifications based on benchmark results and industry standards for API gateway deployments.
### PostgreSQL
Required for authentication, key management, and usage tracking.
| Workload | CPU | RAM | Storage | Connections |
|----------|-----|-----|---------|-------------|
| 1-2K RPS | 4-8 cores | 16GB | 200GB SSD (3000+ IOPS) | 100-200 |
| 2-5K RPS | 8 cores | 16-32GB | 500GB SSD (5000+ IOPS) | 200-500 |
| 5K+ RPS | 16+ cores | 32-64GB | 1TB+ SSD (10000+ IOPS) | 500+ |
**Configuration:** Set `proxy_batch_write_at: 60` to batch writes and reduce DB load. Total connections = pool limit × instances.
### Redis (Recommended)
Redis was not used in these benchmarks but provides significant production benefits: 60-80% reduced DB load.
| Workload | CPU | RAM |
|----------|-----|-----|
| 1-2K RPS | 2-4 cores | 8GB |
| 2-5K RPS | 4 cores | 16GB |
| 5K+ RPS | 8+ cores | 32GB+ |
**Requirements:** Redis 7.0+, AOF persistence enabled, `allkeys-lru` eviction policy.
**Configuration:**
```yaml
router_settings:
redis_host: os.environ/REDIS_HOST
redis_port: os.environ/REDIS_PORT
redis_password: os.environ/REDIS_PASSWORD
litellm_settings:
cache: True
cache_params:
type: redis
host: os.environ/REDIS_HOST
port: os.environ/REDIS_PORT
password: os.environ/REDIS_PASSWORD
```
:::tip
Use `redis_host`, `redis_port`, and `redis_password` instead of `redis_url` for ~80 RPS better performance.
:::
**Scaling:** DB connections scale linearly with instances. Consider PostgreSQL read replicas beyond 5K RPS.
See [Production Configuration](./proxy/prod) for detailed best practices.
## Locust Settings
- 1000 Users
@ -172,7 +224,7 @@ class MyUser(HttpUser):
## Logging Callbacks
### [GCS Bucket Logging](https://docs.litellm.ai/docs/proxy/bucket)
### [GCS Bucket Logging](https://docs.litellm.ai/docs/observability/gcs_bucket_integration)
Using GCS Bucket has **no impact on latency, RPS compared to Basic Litellm Proxy**

2
docs/my-website/docs/image_edits.md
View file

@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| Supported operations | Create image edits | Single and multiple images supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
| Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **Stability AI**, **AWS Bedrock (Stability)** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. Stability AI and Bedrock Stability support various image editing operations. |
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)

2
docs/my-website/docs/index.md
View file

@ -657,7 +657,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml --detailed_debug
```

214
docs/my-website/docs/interactions.md Normal file
View file

@ -0,0 +1,214 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# /interactions
| Feature | Supported | Notes |
|---------|-----------|-------|
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | |
| Loadbalancing | ✅ | Between supported models |
| Supported Providers | `gemini` | [Google Interactions API](https://ai.google.dev/gemini-api/docs/interactions) |
## **LiteLLM Python SDK Usage**
### Quick Start
```python showLineNumbers title="Create Interaction"
from litellm import create_interaction
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = create_interaction(
model="gemini/gemini-2.5-flash",
input="Tell me a short joke about programming."
)
print(response.outputs[-1].text)
```
### Async Usage
```python showLineNumbers title="Async Create Interaction"
from litellm import acreate_interaction
import os
import asyncio
os.environ["GEMINI_API_KEY"] = "your-api-key"
async def main():
response = await acreate_interaction(
model="gemini/gemini-2.5-flash",
input="Tell me a short joke about programming."
)
print(response.outputs[-1].text)
asyncio.run(main())
```
### Streaming
```python showLineNumbers title="Streaming Interaction"
from litellm import create_interaction
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = create_interaction(
model="gemini/gemini-2.5-flash",
input="Write a 3 paragraph story about a robot.",
stream=True
)
for chunk in response:
print(chunk)
```
## **LiteLLM AI Gateway (Proxy) Usage**
### Setup
Add this to your litellm proxy config.yaml:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gemini-flash
litellm_params:
model: gemini/gemini-2.5-flash
api_key: os.environ/GEMINI_API_KEY
```
Start litellm:
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### Test Request
<Tabs>
<TabItem value="curl" label="curl">
```bash showLineNumbers title="Create Interaction"
curl -X POST "http://localhost:4000/v1beta/interactions" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gemini/gemini-2.5-flash",
"input": "Tell me a short joke about programming."
}'
```
**Streaming:**
```bash showLineNumbers title="Streaming Interaction"
curl -N -X POST "http://localhost:4000/v1beta/interactions" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gemini/gemini-2.5-flash",
"input": "Write a 3 paragraph story about a robot.",
"stream": true
}'
```
**Get Interaction:**
```bash showLineNumbers title="Get Interaction by ID"
curl "http://localhost:4000/v1beta/interactions/{interaction_id}" \
-H "Authorization: Bearer sk-1234"
```
</TabItem>
<TabItem value="google-sdk" label="Google GenAI SDK">
Point the Google GenAI SDK to LiteLLM Proxy:
```python showLineNumbers title="Google GenAI SDK with LiteLLM Proxy"
from google import genai
import os
# Point SDK to LiteLLM Proxy
os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
os.environ["GEMINI_API_KEY"] = "sk-1234" # Your LiteLLM API key
client = genai.Client()
# Create an interaction
interaction = client.interactions.create(
model="gemini/gemini-2.5-flash",
input="Tell me a short joke about programming."
)
print(interaction.outputs[-1].text)
```
**Streaming:**
```python showLineNumbers title="Google GenAI SDK Streaming"
from google import genai
import os
os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
os.environ["GEMINI_API_KEY"] = "sk-1234"
client = genai.Client()
for chunk in client.interactions.create_stream(
model="gemini/gemini-2.5-flash",
input="Write a story about space exploration.",
):
print(chunk)
```
</TabItem>
</Tabs>
## **Request/Response Format**
### Request Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | Model to use (e.g., `gemini/gemini-2.5-flash`) |
| `input` | string | Yes | The input text for the interaction |
| `stream` | boolean | No | Enable streaming responses |
| `tools` | array | No | Tools available to the model |
| `system_instruction` | string | No | System instructions for the model |
| `generation_config` | object | No | Generation configuration |
| `previous_interaction_id` | string | No | ID of previous interaction for context |
### Response Format
```json
{
"id": "interaction_abc123",
"object": "interaction",
"model": "gemini-2.5-flash",
"status": "completed",
"created": "2025-01-15T10:30:00Z",
"updated": "2025-01-15T10:30:05Z",
"role": "model",
"outputs": [
{
"type": "text",
"text": "Why do programmers prefer dark mode? Because light attracts bugs!"
}
],
"usage": {
"total_input_tokens": 10,
"total_output_tokens": 15,
"total_tokens": 25
}
}
```
## **Supported Providers**
| Provider | Link to Usage |
|----------|---------------|
| Google AI Studio | [Usage](#quick-start) |

238
docs/my-website/docs/observability/azure_sentinel.md Normal file
View file

@ -0,0 +1,238 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Azure Sentinel
<Image img={require('../../img/sentinel.png')} />
LiteLLM supports logging to Azure Sentinel via the Azure Monitor Logs Ingestion API. Azure Sentinel uses Log Analytics workspaces for data storage, so logs sent to the workspace will be available in Sentinel for security monitoring and analysis.
## Azure Sentinel Integration
| Feature | Details |
|---------|---------|
| **What is logged** | [StandardLoggingPayload](../proxy/logging_spec) |
| **Events** | Success + Failure |
| **Product Link** | [Azure Sentinel](https://learn.microsoft.com/en-us/azure/sentinel/overview) |
| **API Reference** | [Logs Ingestion API](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/logs-ingestion-api-overview) |
We will use the `--config` to set `litellm.callbacks = ["azure_sentinel"]` this will log all successful and failed LLM calls to Azure Sentinel.
**Step 1**: Create a `config.yaml` file and set `litellm_settings`: `callbacks`
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
litellm_settings:
callbacks: ["azure_sentinel"] # logs llm success + failure logs to Azure Sentinel
```
**Step 2**: Set Up Azure Resources
Before using the Logs Ingestion API, you need to set up the following in Azure:
1. **Create a Log Analytics Workspace** (if you don't have one)
2. **Create a Custom Table** in your Log Analytics workspace (e.g., `LiteLLM_CL`)
3. **Create a Data Collection Rule (DCR)** with:
- Stream declaration matching your data structure
- Transformation to map data to your custom table
- Access granted to your app registration
4. **Register an Application** in Microsoft Entra ID (Azure AD) with:
- Client ID
- Client Secret
- Permissions to write to the DCR
For detailed setup instructions, see the [Microsoft documentation on Logs Ingestion API](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/logs-ingestion-api-overview).
**Step 3**: Set Required Environment Variables
Set the following environment variables with your Azure credentials:
```shell showLineNumbers title="Environment Variables"
# Required: Data Collection Rule (DCR) configuration
AZURE_SENTINEL_DCR_IMMUTABLE_ID="dcr-xxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # DCR Immutable ID from Azure portal
AZURE_SENTINEL_STREAM_NAME="Custom-LiteLLM_CL_CL" # Stream name from your DCR
AZURE_SENTINEL_ENDPOINT="https://your-dcr-endpoint.eastus-1.ingest.monitor.azure.com" # DCR logs ingestion endpoint (NOT the DCE endpoint)
# Required: OAuth2 Authentication (App Registration)
AZURE_SENTINEL_TENANT_ID="your-tenant-id" # Azure Tenant ID
AZURE_SENTINEL_CLIENT_ID="your-client-id" # Application (client) ID
AZURE_SENTINEL_CLIENT_SECRET="your-client-secret" # Client secret value
```
**Note**: The `AZURE_SENTINEL_ENDPOINT` should be the DCR's logs ingestion endpoint (found in the DCR Overview page), NOT the Data Collection Endpoint (DCE). The DCR endpoint is associated with your specific DCR and looks like: `https://your-dcr-endpoint.{region}-1.ingest.monitor.azure.com`
**Step 4**: Start the proxy and make a test request
Start proxy
```shell showLineNumbers title="Start Proxy"
litellm --config config.yaml --debug
```
Test Request
```shell showLineNumbers title="Test Request"
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
"metadata": {
"your-custom-metadata": "custom-field",
}
}'
```
**Step 5**: View logs in Azure Sentinel
1. Navigate to your Azure Sentinel workspace in the Azure portal
2. Go to "Logs" and query your custom table (e.g., `LiteLLM_CL`)
3. Run a query like:
```kusto showLineNumbers title="KQL Query"
LiteLLM_CL
| where TimeGenerated > ago(1h)
| project TimeGenerated, model, status, total_tokens, response_cost
| order by TimeGenerated desc
```
You should see following logs in Azure Workspace.
<Image img={require('../../img/sentinel.png')} />
## Environment Variables
| Environment Variable | Description | Default Value | Required |
|---------------------|-------------|---------------|----------|
| `AZURE_SENTINEL_DCR_IMMUTABLE_ID` | Data Collection Rule (DCR) Immutable ID | None | ✅ Yes |
| `AZURE_SENTINEL_ENDPOINT` | DCR logs ingestion endpoint URL (from DCR Overview page) | None | ✅ Yes |
| `AZURE_SENTINEL_STREAM_NAME` | Stream name from DCR (e.g., "Custom-LiteLLM_CL_CL") | "Custom-LiteLLM" | ❌ No |
| `AZURE_SENTINEL_TENANT_ID` | Azure Tenant ID for OAuth2 authentication | None (falls back to `AZURE_TENANT_ID`) | ✅ Yes |
| `AZURE_SENTINEL_CLIENT_ID` | Application (client) ID for OAuth2 authentication | None (falls back to `AZURE_CLIENT_ID`) | ✅ Yes |
| `AZURE_SENTINEL_CLIENT_SECRET` | Client secret for OAuth2 authentication | None (falls back to `AZURE_CLIENT_SECRET`) | ✅ Yes |
## How It Works
The Azure Sentinel integration uses the [Azure Monitor Logs Ingestion API](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/logs-ingestion-api-overview) to send logs to your Log Analytics workspace. The integration:
- Authenticates using OAuth2 client credentials flow with your app registration
- Sends logs to the Data Collection Rule (DCR) endpoint
- Batches logs for efficient transmission
- Sends logs in the [StandardLoggingPayload](../proxy/logging_spec) format
- Automatically handles both success and failure events
- Caches OAuth2 tokens and refreshes them automatically
Logs sent to the Log Analytics workspace are automatically available in Azure Sentinel for security monitoring, threat detection, and analysis.
## Azure Sentinel Setup Guide
Follow this step-by-step guide to set up Azure Sentinel with LiteLLM.
### Step 1: Create a Log Analytics Workspace
1. Navigate to [https://portal.azure.com/#home](https://portal.azure.com/#home)
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/5659f6f5-a166-4b26-a991-73352274e3bb/ascreenshot.jpeg?tl_px=0,210&br_px=2618,1673&force_format=jpeg&q=100&width=1120.0)
2. Search for "Log Analytics workspaces" and click "Create"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/a827ba10-a391-486a-a36a-51816c6255de/ascreenshot.jpeg?tl_px=0,0&br_px=2618,1463&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=21,106)
3. Enter a name for your workspace (e.g., "litellm-sentinel-prod")
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/943458f1-fd4c-47dd-a273-ea5a04734ed9/ascreenshot.jpeg?tl_px=0,420&br_px=2618,1884&force_format=jpeg&q=100&width=1120.0)
4. Click "Review + Create"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/c54828fb-f895-4eb7-b810-cacf437617bd/ascreenshot.jpeg?tl_px=0,420&br_px=2618,1884&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=40,564)
### Step 2: Create a Custom Table
1. Go to your Log Analytics workspace and click "Tables"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/72d65f70-75c0-471f-95e9-947c72e173cc/ascreenshot.jpeg?tl_px=0,142&br_px=2618,1605&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=330,277)
2. Click "Create" → "New custom log (Direct Ingest)"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/863ad29b-2c3a-4b7c-9a6b-36d3a76c9f32/ascreenshot.jpeg?tl_px=0,0&br_px=2618,1463&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=526,146)
3. Enter a table name (e.g., "LITELLM_PROD_CL")
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/ef2f1c52-aa36-46a1-91e6-9bd868891b15/ascreenshot.jpeg?tl_px=0,0&br_px=2618,1463&force_format=jpeg&q=100&width=1120.0)
### Step 3: Create a Data Collection Rule (DCR)
1. Click "Create a new data collection rule"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/f2abc0d3-8be8-4057-9290-946d10cfd183/ascreenshot.jpeg?tl_px=0,420&br_px=2618,1884&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=264,404)
2. Enter a name for the DCR (e.g., "litellm-prod")
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/79bbebdc-e4d9-46ff-a270-1930619050a1/ascreenshot.jpeg?tl_px=0,8&br_px=2618,1471&force_format=jpeg&q=100&width=1120.0)
3. Select a Data Collection Endpoint
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/f3112e9a-551e-415c-a7f9-55aad801bc8a/ascreenshot.jpeg?tl_px=0,420&br_px=2618,1884&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=332,480)
4. Upload the sample JSON file for schema (use the [example_standard_logging_payload.json](https://github.com/BerriAI/litellm/blob/main/litellm/integrations/azure_sentinel/example_standard_logging_payload.json) file)
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/703c0762-840a-4f1f-a60f-876dc24b7a03/ascreenshot.jpeg?tl_px=0,0&br_px=2618,1463&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=518,272)
5. Click "Next" and then "Create"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/0bca0200-5c64-4fbd-8061-9308aa6656b8/ascreenshot.jpeg?tl_px=0,420&br_px=2618,1884&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=128,560)
### Step 4: Get the DCR Immutable ID and Logs Ingestion Endpoint
1. Go to "Data Collection Rules" and select your DCR
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/11c06a0d-584f-4d22-b36e-9c338d43812c/ascreenshot.jpeg?tl_px=0,0&br_px=2618,1463&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=94,258)
2. Copy the **DCR Immutable ID** (starts with `dcr-`)
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/cd0ad69a-4d95-4b6a-9533-7720908ba809/ascreenshot.jpeg?tl_px=1160,92&br_px=2618,907&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=530,277)
3. Copy the **Logs Ingestion Endpoint** URL
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/3d3752ed-08ea-4490-8c98-a97d33947ea7/ascreenshot.jpeg?tl_px=1160,464&br_px=2618,1279&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=532,277)
### Step 5: Get the Stream Name
1. Click "JSON View" in the DCR
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/fd8a5504-4769-4f23-983e-520f256ee308/ascreenshot.jpeg?tl_px=1160,0&br_px=2618,814&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=965,257)
2. Find the **Stream Name** in the `streamDeclarations` section (e.g., "Custom-LITELLM_PROD_CL_CL")
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-17/a4052b32-2028-4d12-8930-bfcdf6f47652/ascreenshot.jpeg?tl_px=405,270&br_px=2115,1225&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=523,277)
### Step 6: Register an App and Grant Permissions
1. Go to **Microsoft Entra ID** → **App registrations** → **New registration**
2. Create a new app and note the **Client ID** and **Tenant ID**
3. Go to **Certificates & secrets** → Create a new client secret and copy the **Secret Value**
4. Go back to your DCR → **Access Control (IAM)** → **Add role assignment**
5. Assign the **"Monitoring Metrics Publisher"** role to your app registration
### Summary: Where to Find Each Value
| Environment Variable | Where to Find It |
|---------------------|------------------|
| `AZURE_SENTINEL_DCR_IMMUTABLE_ID` | DCR Overview page → Immutable ID (starts with `dcr-`) |
| `AZURE_SENTINEL_ENDPOINT` | DCR Overview page → Logs Ingestion Endpoint |
| `AZURE_SENTINEL_STREAM_NAME` | DCR JSON View → `streamDeclarations` section |
| `AZURE_SENTINEL_TENANT_ID` | App Registration → Overview → Directory (tenant) ID |
| `AZURE_SENTINEL_CLIENT_ID` | App Registration → Overview → Application (client) ID |
| `AZURE_SENTINEL_CLIENT_SECRET` | App Registration → Certificates & secrets → Secret Value |
For more details, refer to the [Microsoft Logs Ingestion API documentation](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/logs-ingestion-api-overview).

2
docs/my-website/docs/observability/datadog.md
View file

@ -181,7 +181,7 @@ docker run \
-e USE_DDTRACE=true \
-e USE_DDPROFILER=true \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml --detailed_debug
```

2
docs/my-website/docs/oidc.md
View file

@ -106,7 +106,7 @@ model_list:
aws_region_name: us-west-2
aws_session_name: "my-test-session"
aws_role_name: "arn:aws:iam::335785316107:role/litellm-github-unit-tests-circleci"
aws_web_identity_token: "oidc/circleci_v2/"
aws_web_identity_token: "oidc/example-provider/"
```
#### Amazon IAM Role Configuration for CircleCI v2 -> Bedrock

84
docs/my-website/docs/providers/anthropic.md
View file

@ -1936,3 +1936,87 @@ curl http://0.0.0.0:4000/v1/chat/completions \
</TabItem>
</Tabs>
## Usage - Agent Skills
LiteLLM supports using Agent Skills with the API
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = completion(
model="claude-sonnet-4-5-20250929",
messages=messages,
tools= [
{
"type": "code_execution_20250825",
"name": "code_execution"
}
],
container= {
"skills": [
{
"type": "anthropic",
"skill_id": "pptx",
"version": "latest"
}
]
}
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: claude-sonnet-4-5-20250929
litellm_params:
model: anthropic/claude-sonnet-4-5-20250929
api_key: os.environ/ANTHROPIC_API_KEY
```
2. Start Proxy
```
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl --location 'http://localhost:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer <YOUR-LITELLM-KEY>' \
--data '{
"model": "claude-sonnet-4-5-20250929",
"messages": [
{
"role": "user",
"content": "Hi"
}
],
"tools": [
{
"type": "code_execution_20250825",
"name": "code_execution"
}
],
"container": {
"skills": [
{
"type": "anthropic",
"skill_id": "pptx",
"version": "latest"
}
]
}
}'
```
</TabItem>
</Tabs>
The container and its "id" will be present in "provider_specific_fields" in streaming/non-streaming response

93
docs/my-website/docs/providers/azure_ai_agents.md
View file

@ -328,7 +328,100 @@ model_list:
| `thread_id` | string | Optional thread ID to continue an existing conversation |
| `instructions` | string | Optional instructions to override the agent's default instructions for this run |
## LiteLLM A2A Gateway
You can also connect to Azure AI Foundry Agents through LiteLLM's A2A (Agent-to-Agent) Gateway UI. This provides a visual way to register and test agents without writing code.
### 1. Navigate to Agents
From the sidebar, click "Agents" to open the agent management page, then click "+ Add New Agent".
![Add New Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/f8efe335-a08a-4f2b-9f7f-de28e4d58b05/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=217,118)
### 2. Select Azure AI Foundry Agent Type
Click "A2A Standard" to see available agent types, then select "Azure AI Foundry".
![Select A2A Standard](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/ede38044-3e18-43b9-afe3-b7513bf9963e/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=409,143)
![Select Azure AI Foundry](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/33c396fc-a927-4b03-8ee2-ea04950b12c1/ascreenshot.jpeg?tl_px=0,86&br_px=2201,1317&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=433,277)
### 3. Configure the Agent
Fill in the following fields:
#### Agent Name
Enter a friendly agent name - callers will see this name as the agent available.
![Enter Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/18c02804-7612-40c4-9ba4-3f1a4c0725d5/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1463&force_format=jpeg&q=100&width=1120.0)
#### Agent ID
Get the Agent ID from your Azure AI Foundry portal:
1. Go to [https://ai.azure.com/](https://ai.azure.com/) and click "Agents"
![Azure Agents](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/5e29fc48-c0f7-4b6d-8313-2063d1240d15/ascreenshot.jpeg?tl_px=0,0&br_px=2618,1463&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=39,187)
2. Copy the "ID" of the agent you want to add (e.g., `asst_hbnoK9BOCcHhC3lC4MDroVGG`)
![Copy Agent ID](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/bf17dfec-a627-41c6-9121-3935e86d3700/ascreenshot.jpeg?tl_px=0,0&br_px=2618,1463&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=504,241)
3. Paste the Agent ID in LiteLLM - this tells LiteLLM which agent to invoke on Azure Foundry
![Paste Agent ID](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/45230c28-54f6-441c-9a20-4ef8b74076e2/ascreenshot.jpeg?tl_px=0,97&br_px=2617,1560&force_format=jpeg&q=100&width=1120.0)
#### Azure AI API Base
Get your API base URL from Azure AI Foundry:
1. Go to [https://ai.azure.com/](https://ai.azure.com/) and click "Overview"
2. Under libraries, select Microsoft Foundry
3. Get your endpoint - it should look like `https://<domain>.services.ai.azure.com/api/projects/<project-name>`
![Get API Base](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/60e2c735-4480-44b7-ab12-d69f4200b12c/ascreenshot.jpeg?tl_px=0,40&br_px=2618,1503&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=278,277)
4. Paste the URL in LiteLLM
![Paste API Base](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/e9c6f48e-7602-449a-9261-0df4a0a66876/ascreenshot.jpeg?tl_px=267,456&br_px=2468,1687&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,277)
#### Authentication
Add your Azure AD credentials for authentication:
- **Azure Tenant ID**
- **Azure Client ID**
- **Azure Client Secret**
![Add Auth](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/e5e2b636-cf2e-4283-a1cc-8d497d349243/ascreenshot.jpeg?tl_px=0,653&br_px=2201,1883&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=339,405)
Click "Create Agent" to save.
![Create Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/799a720a-639e-4217-a6f5-51687fc07611/ascreenshot.jpeg?tl_px=416,653&br_px=2618,1883&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=693,519)
### 4. Test in Playground
Go to "Playground" in the sidebar to test your agent.
![Go to Playground](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/7da84247-db1c-4d55-9015-6e3d60ea63ce/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=63,106)
Change the endpoint type to `/v1/a2a/message/send`.
![Select A2A Endpoint](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/733265a8-412d-4eac-bc19-03436d7846c4/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=286,234)
### 5. Select Your Agent and Send a Message
Pick your Azure AI Foundry agent from the dropdown and send a test message.
![Select Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/59a8e66e-6f82-42e3-ab48-78355464e6be/ascreenshot.jpeg?tl_px=0,28&br_px=2201,1259&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=269,277)
The agent responds with its capabilities. You can now interact with your Azure AI Foundry agent through the A2A protocol.
![Agent Response](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-14/a0aafb69-6c28-4977-8210-96f9de750cdf/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=487,272)
## Further Reading
- [Azure AI Foundry Agents Documentation](https://learn.microsoft.com/en-us/azure/ai-services/agents/)
- [Create Thread and Run API Reference](https://learn.microsoft.com/en-us/rest/api/aifoundry/aiagents/create-thread-and-run/create-thread-and-run)
- [A2A Agent Gateway](../a2a.md)
- [A2A Cost Tracking](../a2a_cost_tracking.md)

108
docs/my-website/docs/providers/custom_llm_server.md
View file

@ -17,6 +17,7 @@ Supported Routes:
- `/v1/completions` -> `litellm.atext_completion`
- `/v1/embeddings` -> `litellm.aembedding`
- `/v1/images/generations` -> `litellm.aimage_generation`
- `/v1/images/edits` -> `litellm.aimage_edit`
- `/v1/messages` -> `litellm.acompletion`
@ -263,6 +264,83 @@ Expected Response
}
```
## Image Edit
1. Setup your `custom_handler.py` file
```python
import litellm
from litellm import CustomLLM
from litellm.types.utils import ImageResponse, ImageObject
import time
class MyCustomLLM(CustomLLM):
async def aimage_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
# Your custom image edit logic here
# e.g., call Stability AI, Black Forest Labs, etc.
return ImageResponse(
created=int(time.time()),
data=[ImageObject(url="https://example.com/edited-image.png")],
)
my_custom_llm = MyCustomLLM()
```
2. Add to `config.yaml`
In the config below, we pass
python_filename: `custom_handler.py`
custom_handler_instance_name: `my_custom_llm`. This is defined in Step 1
custom_handler: `custom_handler.my_custom_llm`
```yaml
model_list:
- model_name: "my-custom-image-edit-model"
litellm_params:
model: "my-custom-llm/my-model"
litellm_settings:
custom_provider_map:
- {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
```
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/v1/images/edits' \
-H 'Authorization: Bearer sk-1234' \
-F 'model=my-custom-image-edit-model' \
-F 'image=@/path/to/image.png' \
-F 'prompt=Make the sky blue'
```
Expected Response
```
{
"created": 1721955063,
"data": [{"url": "https://example.com/edited-image.png"}],
}
```
## Anthropic `/v1/messages`
- Write the integration for .acompletion
@ -517,4 +595,34 @@ class CustomLLM(BaseLLM):
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
def image_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[HTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
async def aimage_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
```

45
docs/my-website/docs/providers/fireworks_ai.md
View file

@ -300,6 +300,51 @@ litellm_settings:
</TabItem>
</Tabs>
## Reasoning Effort
The `reasoning_effort` parameter is supported on select Fireworks AI models. Supported models include:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ["FIREWORKS_AI_API_KEY"] = "YOUR_API_KEY"
response = completion(
model="fireworks_ai/accounts/fireworks/models/qwen3-8b",
messages=[
{"role": "user", "content": "What is the capital of France?"}
],
reasoning_effort="low",
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "fireworks_ai/accounts/fireworks/models/qwen3-8b",
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
],
"reasoning_effort": "low"
}'
```
</TabItem>
</Tabs>
## Supported Models - ALL Fireworks AI Models Supported!
:::info

53
docs/my-website/docs/providers/openai/responses_api.md
View file

@ -623,6 +623,58 @@ display(styled_df)
</TabItem>
</Tabs>
## Function Calling
```python showLineNumbers title="Function Calling with Parallel Tool Calls"
import litellm
import json
tools = [
{
"type": "function",
"name": "get_weather",
"description": "Get current weather for a location",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
]
# Step 1: Request with tools (parallel_tool_calls=True allows multiple calls)
response = litellm.responses(
model="openai/gpt-4o",
input=[{"role": "user", "content": "What's the weather in Paris and Tokyo?"}],
tools=tools,
parallel_tool_calls=True, # Defaults = True
)
# Step 2: Execute tool calls and collect results
tool_results = []
for output in response.output:
if output.type == "function_call":
result = {"temperature": 15, "condition": "sunny"} # Your function logic here
tool_results.append({
"type": "function_call_output",
"call_id": output.call_id,
"output": json.dumps(result)
})
# Step 3: Send results back
final_response = litellm.responses(
model="openai/gpt-4o",
input=tool_results,
tools=tools,
)
print(final_response.output)
```
Set `parallel_tool_calls=False` to ensure zero or one tool is called per turn. [More details](https://platform.openai.com/docs/guides/function-calling#parallel-function-calling).
## Free-form Function Calling
<Tabs>
@ -633,7 +685,6 @@ display(styled_df)
import litellm
response = litellm.responses(
response = client.responses.create(
model="gpt-5-mini",
input="Please use the code_exec tool to calculate the area of a circle with radius equal to the number of 'r's in strawberry",
text={"format": {"type": "text"}},

121
docs/my-website/docs/providers/pydantic_ai_agent.md Normal file
View file

@ -0,0 +1,121 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Pydantic AI Agents
Call Pydantic AI Agents via LiteLLM's A2A Gateway.
| Property | Details |
|----------|---------|
| Description | Pydantic AI agents with native A2A support via the `to_a2a()` method. LiteLLM provides fake streaming support for agents that don't natively stream. |
| Provider Route on LiteLLM | A2A Gateway |
| Supported Endpoints | `/v1/a2a/message/send` |
| Provider Doc | [Pydantic AI Agents ↗](https://ai.pydantic.dev/agents/) |
## LiteLLM A2A Gateway
All Pydantic AI agents need to be exposed as A2A agents using the `to_a2a()` method. Once your agent server is running, you can add it to the LiteLLM Gateway.
### 1. Setup Pydantic AI Agent Server
LiteLLM requires Pydantic AI agents to follow the [A2A (Agent-to-Agent) protocol](https://github.com/google/A2A). Pydantic AI has native A2A support via the `to_a2a()` method, which exposes your agent as an A2A-compliant server.
#### Install Dependencies
```bash
pip install pydantic-ai fasta2a uvicorn
```
#### Create Agent
```python title="agent.py"
from pydantic_ai import Agent
agent = Agent('openai:gpt-4o-mini', instructions='Be helpful!')
@agent.tool_plain
def get_weather(city: str) -> str:
"""Get weather for a city."""
return f"Weather in {city}: Sunny, 72°F"
@agent.tool_plain
def calculator(expression: str) -> str:
"""Evaluate a math expression."""
return str(eval(expression))
# Native A2A server - Pydantic AI handles it automatically
app = agent.to_a2a()
```
#### Run Server
```bash
uvicorn agent:app --host 0.0.0.0 --port 9999
```
Server runs at `http://localhost:9999`
### 2. Navigate to Agents
From the sidebar, click "Agents" to open the agent management page, then click "+ Add New Agent".
### 3. Select Pydantic AI Agent Type
Click "A2A Standard" to see available agent types, then select "Pydantic AI".
![Select A2A Standard](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/1055acb1-064b-4465-8e6a-8278291bc661/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=395,147)
![Select Pydantic AI](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/0998e38c-8534-40f1-931a-be96c2cae0ad/ascreenshot.jpeg?tl_px=0,52&br_px=2201,1283&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=421,277)
### 4. Configure the Agent
Fill in the following fields:
- **Agent Name** - A unique identifier for your agent (e.g., `test-pydantic-agent`)
- **Agent URL** - The URL where your Pydantic AI agent is running. We use `http://localhost:9999` because that's where we started our Pydantic AI agent server in the previous step.
![Enter Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/8cf3fbde-05f3-48d1-81b6-6f857bd6d360/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=443,225)
![Configure Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fb555808-4761-4c49-a415-200ac1bdb525/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1463&force_format=jpeg&q=100&width=1120.0)
![Enter Agent URL](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/303eae61-4352-4fb0-a537-806839c234ba/ascreenshot.jpeg?tl_px=0,212&br_px=2201,1443&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=456,277)
### 5. Create Agent
Click "Create Agent" to save your configuration.
![Create Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/914f3367-df7d-4244-bd4d-e99ce0a6193a/ascreenshot.jpeg?tl_px=416,438&br_px=2618,1669&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=690,277)
### 6. Test in Playground
Go to "Playground" in the sidebar to test your agent.
![Go to Playground](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/c73c9f3b-22af-4105-aafa-2d34c4986ef3/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=44,97)
### 7. Select A2A Endpoint
Click the endpoint dropdown and search for "a2a", then select `/v1/a2a/message/send`.
![Click Endpoint Dropdown](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/196d97ac-bcba-47f0-9880-97b80250e00c/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=261,230)
![Search for A2A](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/26b68f21-29f9-4c4c-b8b5-d2e11cbfd14a/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1463&force_format=jpeg&q=100&width=1120.0)
![Select A2A Endpoint](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/41576fb1-d385-4fb2-84e9-142dd7fe5181/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=307,270)
### 8. Select Your Agent and Send a Message
Pick your Pydantic AI agent from the dropdown and send a test message.
![Click Agent Dropdown](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a96d7967-3d54-4cbf-bd3e-b38f1be9df76/ascreenshot.jpeg?tl_px=0,54&br_px=2201,1285&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=274,277)
![Select Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/e05a5a6e-d044-4480-b94e-7c03cfb92ac5/ascreenshot.jpeg?tl_px=0,113&br_px=2201,1344&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=290,277)
![Send Message](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/29162702-968a-401a-aac1-c844bfc5f4a3/ascreenshot.jpeg?tl_px=91,653&br_px=2292,1883&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,436)
## Further Reading
- [Pydantic AI Documentation](https://ai.pydantic.dev/)
- [Pydantic AI Agents](https://ai.pydantic.dev/agents/)
- [A2A Agent Gateway](../a2a.md)
- [A2A Cost Tracking](../a2a_cost_tracking.md)

65
docs/my-website/docs/providers/sap.md
View file

@ -5,12 +5,12 @@ import TabItem from '@theme/TabItem';
LiteLLM supports SAP Generative AI Hub's Orchestration Service.
| Property | Details |
|-------|-------|
| Description | SAP's Generative AI Hub provides access to foundation models through the AI Core orchestration service. |
| Provider Route on LiteLLM | `sap/` |
| Supported Endpoints | `/chat/completions` |
| API Reference | [SAP AI Core Documentation](https://help.sap.com/docs/sap-ai-core) |
| Property | Details |
|-------|--------------------------------------------------------------------------------------------------------------------------------------------------------|
| Description | SAP's Generative AI Hub provides access to OpenAI, Anthropic, Gemini, Mistral, NVIDIA, Amazon, and SAP LLMs through the AI Core orchestration service. |
| Provider Route on LiteLLM | `sap/` |
| Supported Endpoints | `/chat/completions`, `/embeddings` |
| API Reference | [SAP AI Core Documentation](https://help.sap.com/docs/sap-ai-core) |
## Authentication
@ -23,7 +23,14 @@ SAP Generative AI Hub uses service key authentication. You can provide credentia
import os
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
```
3. **Environment variables** - Set the following list of credentials in .env file
<pre>
AICORE_AUTH_URL = "https://* * * .authentication.sap.hana.ondemand.com/oauth/token",
AICORE_CLIENT_ID = " *** ",
AICORE_CLIENT_SECRET = " *** ",
AICORE_RESOURCE_GROUP = " *** ",
AICORE_BASE_URL = "https://api.ai.***.cfapps.sap.hana.ondemand.com/v2"
</pre>
## Usage - LiteLLM Python SDK
```python showLineNumbers title="SAP Chat Completion"
@ -55,16 +62,33 @@ for chunk in response:
print(chunk.choices[0].delta.content or "", end="")
```
```python showLineNumbers title="SAP Embedding"
from litellm import embedding
import os
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
result = embedding(
model="sap/text-embedding-3-small",
input="Answer to the ultimate question of life, the universe, and everything is 42")
print(result.data[0])
```
## Usage - LiteLLM Proxy
Add to your LiteLLM Proxy config:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: sap-gpt4
- model_name: "sap/*"
litellm_params:
model: sap/gpt-4
api_key: os.environ/AICORE_SERVICE_KEY
model: "sap/*"
general_settings:
master_key: your-proxy-api-key
environment_variables:
AICORE_SERVICE_KEY: '{"clientid": "...", "clientsecret": "...", ...}'
```
Start the proxy:
@ -81,7 +105,7 @@ curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-d '{
"model": "sap-gpt4",
"model": "sap/gpt-4",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
@ -98,12 +122,29 @@ client = OpenAI(
)
response = client.chat.completions.create(
model="sap-gpt4",
model="sap/gpt-4",
messages=[{"role": "user", "content": "Hello"}]
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="litellm-sdk" label="LiteLLM SDK">
```python showLineNumbers title="LiteLLM SDK"
import os
import litellm
os.environ["LITELLM_PROXY_API_KEY"] = "your-proxy-api-key"
litellm.use_litellm_proxy = True # it is important to set this parameter
response = litellm.completion(
model="sap/gpt-4o",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="http://your-proxy-api-base"
)
print(response)
```
</TabItem>
</Tabs>

284
docs/my-website/docs/providers/stability.md
View file

@ -8,7 +8,7 @@ https://stability.ai/
| Description | Stability AI creates open AI models for image, video, audio, and 3D generation. Known for Stable Diffusion. |
| Provider Route on LiteLLM | `stability/` |
| Link to Provider Doc | [Stability AI API ↗](https://platform.stability.ai/docs/api-reference) |
| Supported Operations | [`/images/generations`](#image-generation) |
| Supported Operations | [`/images/generations`](#image-generation), [`/images/edits`](#image-editing) |
LiteLLM supports Stability AI Image Generation calls via the Stability AI REST API (not via Bedrock).
@ -169,13 +169,285 @@ Stability AI returns images in base64 format. The response is OpenAI-compatible:
}
```
## Comparing with Bedrock
## Image Editing
Stability AI supports various image editing operations including inpainting, upscaling, outpainting, background removal, and more.
### Usage - LiteLLM Python SDK
#### Inpainting (Edit with Mask)
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Inpainting - edit specific areas using a mask
response = image_edit(
model="stability/stable-image-inpaint-v1:0",
image=open("original_image.png", "rb"),
mask=open("mask_image.png", "rb"),
prompt="Add a beautiful sunset in the masked area",
size="1024x1024",
)
print(response)
```
#### Image Upscaling
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Conservative upscaling - preserves details
response = image_edit(
model="stability/stable-conservative-upscale-v1:0",
image=open("low_res_image.png", "rb"),
prompt="Upscale this image while preserving details",
)
# Creative upscaling - adds creative details
response = image_edit(
model="stability/stable-creative-upscale-v1:0",
image=open("low_res_image.png", "rb"),
prompt="Upscale and enhance with creative details",
creativity=0.3, # 0-0.35, higher = more creative
)
# Fast upscaling - quick upscaling
response = image_edit(
model="stability/stable-fast-upscale-v1:0",
image=open("low_res_image.png", "rb"),
prompt="Quickly upscale this image",
)
print(response)
```
#### Image Outpainting
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Extend image beyond its borders
response = image_edit(
model="stability/stable-outpaint-v1:0",
image=open("original_image.png", "rb"),
prompt="Extend this landscape with mountains",
left=100, # Pixels to extend on the left
right=100, # Pixels to extend on the right
up=50, # Pixels to extend on top
down=50, # Pixels to extend on bottom
)
print(response)
```
#### Background Removal
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Remove background from image
response = image_edit(
model="stability/stable-image-remove-background-v1:0",
image=open("portrait.png", "rb"),
prompt="Remove the background",
)
print(response)
```
#### Search and Replace
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Search and replace objects in image
response = image_edit(
model="stability/stable-image-search-replace-v1:0",
image=open("scene.png", "rb"),
prompt="A red sports car",
search_prompt="blue sedan", # What to replace
)
# Search and recolor
response = image_edit(
model="stability/stable-image-search-recolor-v1:0",
image=open("scene.png", "rb"),
prompt="Make it golden yellow",
select_prompt="the car", # What to recolor
)
print(response)
```
#### Image Control (Sketch/Structure)
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Control with sketch
response = image_edit(
model="stability/stable-image-control-sketch-v1:0",
image=open("sketch.png", "rb"),
prompt="Turn this sketch into a realistic photo",
control_strength=0.7, # 0-1, higher = more control
)
# Control with structure
response = image_edit(
model="stability/stable-image-control-structure-v1:0",
image=open("structure_reference.png", "rb"),
prompt="Generate image following this structure",
control_strength=0.7,
)
print(response)
```
#### Erase Objects
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Erase objects from image
response = image_edit(
model="stability/stable-image-erase-object-v1:0",
image=open("scene.png", "rb"),
mask=open("object_mask.png", "rb"), # Mask the object to erase
prompt="Remove the object",
)
print(response)
```
### Supported Image Edit Models
| Model Name | Function Call | Description |
|------------|---------------|-------------|
| stable-image-inpaint-v1:0 | `image_edit(model="stability/stable-image-inpaint-v1:0", ...)` | Inpainting with mask |
| stable-conservative-upscale-v1:0 | `image_edit(model="stability/stable-conservative-upscale-v1:0", ...)` | Conservative upscaling |
| stable-creative-upscale-v1:0 | `image_edit(model="stability/stable-creative-upscale-v1:0", ...)` | Creative upscaling |
| stable-fast-upscale-v1:0 | `image_edit(model="stability/stable-fast-upscale-v1:0", ...)` | Fast upscaling |
| stable-outpaint-v1:0 | `image_edit(model="stability/stable-outpaint-v1:0", ...)` | Extend image borders |
| stable-image-remove-background-v1:0 | `image_edit(model="stability/stable-image-remove-background-v1:0", ...)` | Remove background |
| stable-image-search-replace-v1:0 | `image_edit(model="stability/stable-image-search-replace-v1:0", ...)` | Search and replace objects |
| stable-image-search-recolor-v1:0 | `image_edit(model="stability/stable-image-search-recolor-v1:0", ...)` | Search and recolor |
| stable-image-control-sketch-v1:0 | `image_edit(model="stability/stable-image-control-sketch-v1:0", ...)` | Control with sketch |
| stable-image-control-structure-v1:0 | `image_edit(model="stability/stable-image-control-structure-v1:0", ...)` | Control with structure |
| stable-image-erase-object-v1:0 | `image_edit(model="stability/stable-image-erase-object-v1:0", ...)` | Erase objects |
| stable-image-style-guide-v1:0 | `image_edit(model="stability/stable-image-style-guide-v1:0", ...)` | Apply style guide |
| stable-style-transfer-v1:0 | `image_edit(model="stability/stable-style-transfer-v1:0", ...)` | Transfer style |
### Usage - LiteLLM Proxy Server
#### 1. Setup config.yaml
```yaml showLineNumbers
model_list:
- model_name: stability-inpaint
litellm_params:
model: stability/stable-image-inpaint-v1:0
api_key: os.environ/STABILITY_API_KEY
model_info:
mode: image_edit
- model_name: stability-upscale
litellm_params:
model: stability/stable-conservative-upscale-v1:0
api_key: os.environ/STABILITY_API_KEY
model_info:
mode: image_edit
general_settings:
master_key: sk-1234
```
#### 2. Start the proxy
```bash showLineNumbers
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### 3. Test it
```bash showLineNumbers
curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-H "Authorization: Bearer sk-1234" \
-F "model=stability-inpaint" \
-F "image=@original_image.png" \
-F "mask=@mask_image.png" \
-F "prompt=Add a beautiful garden in the masked area"
```
## AWS Bedrock (Stability)
LiteLLM also supports Stability AI models via AWS Bedrock. This is useful if you're already using AWS infrastructure.
### Usage - Bedrock Stability
```python showLineNumbers
from litellm import image_edit
import os
# Set AWS credentials
os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-key"
os.environ["AWS_REGION_NAME"] = "us-east-1"
# Bedrock Stability inpainting
response = image_edit(
model="bedrock/us.stability.stable-image-inpaint-v1:0",
image=open("original_image.png", "rb"),
mask=open("mask_image.png", "rb"),
prompt="Add flowers in the masked area",
size="1024x1024",
)
print(response)
```
### Supported Bedrock Stability Models
All Stability AI image edit models are available via Bedrock with the `bedrock/` prefix:
| Direct API Model | Bedrock Model | Description |
|------------------|---------------|-------------|
| stability/stable-image-inpaint-v1:0 | bedrock/us.stability.stable-image-inpaint-v1:0 | Inpainting |
| stability/stable-conservative-upscale-v1:0 | bedrock/stability.stable-conservative-upscale-v1:0 | Conservative upscaling |
| stability/stable-creative-upscale-v1:0 | bedrock/stability.stable-creative-upscale-v1:0 | Creative upscaling |
| stability/stable-fast-upscale-v1:0 | bedrock/stability.stable-fast-upscale-v1:0 | Fast upscaling |
| stability/stable-outpaint-v1:0 | bedrock/stability.stable-outpaint-v1:0 | Outpainting |
| stability/stable-image-remove-background-v1:0 | bedrock/stability.stable-image-remove-background-v1:0 | Remove background |
| stability/stable-image-search-replace-v1:0 | bedrock/stability.stable-image-search-replace-v1:0 | Search and replace |
| stability/stable-image-search-recolor-v1:0 | bedrock/stability.stable-image-search-recolor-v1:0 | Search and recolor |
| stability/stable-image-control-sketch-v1:0 | bedrock/stability.stable-image-control-sketch-v1:0 | Control with sketch |
| stability/stable-image-control-structure-v1:0 | bedrock/stability.stable-image-control-structure-v1:0 | Control with structure |
| stability/stable-image-erase-object-v1:0 | bedrock/stability.stable-image-erase-object-v1:0 | Erase objects |
**Note:** Bedrock model IDs may use `us.stability.*` or `stability.*` prefix depending on the region and model.
## Comparing Routes
LiteLLM supports Stability AI models via two routes:
| Route | Provider | Use Case |
|-------|----------|----------|
| `stability/` | Stability AI Direct API | Direct access, all latest models |
| `bedrock/stability.*` | AWS Bedrock | AWS integration, enterprise features |
| Route | Provider | Use Case | Image Generation | Image Editing |
|-------|----------|----------|------------------|---------------|
| `stability/` | Stability AI Direct API | Direct access, all latest models | ✅ | ✅ |
| `bedrock/stability.*` | AWS Bedrock | AWS integration, enterprise features | ✅ | ✅ |
Use `stability/` for direct API access. Use `bedrock/stability.*` if you're already using AWS Bedrock.

216
docs/my-website/docs/providers/vertex_ai_agent_engine.md Normal file
View file

@ -0,0 +1,216 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Vertex AI Agent Engine
Call Vertex AI Agent Engine (Reasoning Engines) in the OpenAI Request/Response format.
| Property | Details |
|----------|---------|
| Description | Vertex AI Agent Engine provides hosted agent runtimes that can execute agentic workflows with foundation models, tools, and custom logic. |
| Provider Route on LiteLLM | `vertex_ai/agent_engine/{RESOURCE_NAME}` |
| Supported Endpoints | `/chat/completions`, `/v1/messages`, `/v1/responses`, `/v1/a2a/message/send` |
| Provider Doc | [Vertex AI Agent Engine ↗](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/overview) |
## Quick Start
### Model Format
```shell showLineNumbers title="Model Format"
vertex_ai/agent_engine/{RESOURCE_NAME}
```
**Example:**
- `vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888`
### LiteLLM Python SDK
```python showLineNumbers title="Basic Agent Completion"
import litellm
response = litellm.completion(
model="vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888",
messages=[
{"role": "user", "content": "Explain machine learning in simple terms"}
],
)
print(response.choices[0].message.content)
```
```python showLineNumbers title="Streaming Agent Responses"
import litellm
response = await litellm.acompletion(
model="vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888",
messages=[
{"role": "user", "content": "What are the key principles of software architecture?"}
],
stream=True,
)
async for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
```
### LiteLLM Proxy
#### 1. Configure your model in config.yaml
<Tabs>
<TabItem value="config-yaml" label="config.yaml">
```yaml showLineNumbers title="LiteLLM Proxy Configuration"
model_list:
- model_name: vertex-agent-1
litellm_params:
model: vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888
vertex_project: your-project-id
vertex_location: us-central1
```
</TabItem>
</Tabs>
#### 2. Start the LiteLLM Proxy
```bash showLineNumbers title="Start LiteLLM Proxy"
litellm --config config.yaml
```
#### 3. Make requests to your Vertex AI Agent Engine
<Tabs>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Basic Agent Request"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "vertex-agent-1",
"messages": [
{"role": "user", "content": "Summarize the main benefits of cloud computing"}
]
}'
```
</TabItem>
<TabItem value="openai-sdk" label="OpenAI Python SDK">
```python showLineNumbers title="Using OpenAI SDK with LiteLLM Proxy"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-litellm-api-key"
)
response = client.chat.completions.create(
model="vertex-agent-1",
messages=[
{"role": "user", "content": "What are best practices for API design?"}
]
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
## LiteLLM A2A Gateway
You can also connect to Vertex AI Agent Engine through LiteLLM's A2A (Agent-to-Agent) Gateway UI. This provides a visual way to register and test agents without writing code.
### 1. Navigate to Agents
From the sidebar, click "Agents" to open the agent management page, then click "+ Add New Agent".
![Click Agents](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9a979927-ce6b-4168-9fba-e53e28f1c2c4/ascreenshot.jpeg?tl_px=0,14&br_px=1376,783&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=17,277)
![Add New Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a311750c-2e85-4589-99cb-2ce7e4021e77/ascreenshot.jpeg?tl_px=0,0&br_px=1376,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=195,257)
### 2. Select Vertex AI Agent Engine Type
Click "A2A Standard" to see available agent types, then select "Vertex AI Agent Engine".
![Select A2A Standard](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/5b1acc4c-dc3f-4639-b4a0-e64b35c228fd/ascreenshot.jpeg?tl_px=52,0&br_px=1428,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,271)
![Select Vertex AI Agent Engine](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/2f3bab61-3e02-4db7-84f0-82200a0f4136/ascreenshot.jpeg?tl_px=0,244&br_px=1376,1013&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=477,277)
### 3. Configure the Agent
Fill in the following fields:
- **Agent Name** - A friendly name for your agent (e.g., `my-vertex-agent`)
- **Reasoning Engine Resource ID** - The full resource path from Google Cloud Console (e.g., `projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888`)
- **Vertex Project** - Your Google Cloud project ID
- **Vertex Location** - The region where your agent is deployed (e.g., `us-central1`)
![Enter Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/695b84c7-9511-4337-bf19-f4505ab2b72b/ascreenshot.jpeg?tl_px=0,90&br_px=1376,859&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=480,276)
![Enter Resource ID](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/ddce64df-b3a3-4519-ab62-f137887bcea2/ascreenshot.jpeg?tl_px=0,294&br_px=1376,1063&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=440,277)
You can find the Resource ID in Google Cloud Console under Vertex AI > Agent Engine:
![Copy Resource ID from Google Cloud Console](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/185d7f17-cbaa-45de-948d-49d2091805ea/ascreenshot.jpeg?tl_px=0,165&br_px=1376,934&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=493,276)
![Enter Vertex Project](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a64da441-3e61-4811-a1e3-9f0b12c949ff/ascreenshot.jpeg?tl_px=0,233&br_px=1376,1002&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=501,277)
You can find the Project ID in Google Cloud Console:
![Copy Project ID from Google Cloud Console](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9ecad3bb-a534-42d6-9604-33906014fad6/user_cropped_screenshot.webp?tl_px=0,0&br_px=1728,1028&force_format=jpeg&q=100&width=1120.0)
![Enter Vertex Location](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/316d1f38-4fb7-4377-86b6-c0fe7ac24383/ascreenshot.jpeg?tl_px=0,330&br_px=1376,1099&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=423,277)
### 4. Create Agent
Click "Create Agent" to save your configuration.
![Create Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fb04b95d-793f-4eed-acf4-d1b3b5fa65e9/ascreenshot.jpeg?tl_px=352,347&br_px=1728,1117&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=623,498)
### 5. Test in Playground
Go to "Playground" in the sidebar to test your agent.
![Go to Playground](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9e01369b-6102-4fe3-96a7-90082cadfd6e/ascreenshot.jpeg?tl_px=0,0&br_px=1376,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=55,226)
### 6. Select A2A Endpoint
Click the endpoint dropdown and select `/v1/a2a/message/send`.
![Select Endpoint](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/d5aeac35-531b-4cf0-af2d-88f0a71fd736/ascreenshot.jpeg?tl_px=0,146&br_px=1376,915&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=299,277)
### 7. Select Your Agent and Send a Message
Pick your Vertex AI Agent Engine from the dropdown and send a test message.
![Select Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/353431f3-a0ba-4436-865d-ae11595e9cc4/ascreenshot.jpeg?tl_px=0,263&br_px=1376,1032&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=270,277)
![Send Message](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fbfce72e-f50b-43e1-b6e5-0d41192d8e2d/ascreenshot.jpeg?tl_px=95,347&br_px=1471,1117&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,474)
![Agent Response](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/892dd826-fbf9-4530-8d82-95270889274a/ascreenshot.jpeg?tl_px=0,82&br_px=1376,851&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=485,277)
## Environment Variables
| Variable | Description |
|----------|-------------|
| `GOOGLE_APPLICATION_CREDENTIALS` | Path to service account JSON key file |
| `VERTEXAI_PROJECT` | Google Cloud project ID |
| `VERTEXAI_LOCATION` | Google Cloud region (default: `us-central1`) |
```bash
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json"
export VERTEXAI_PROJECT="your-project-id"
export VERTEXAI_LOCATION="us-central1"
```
## Further Reading
- [Vertex AI Agent Engine Documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/overview)
- [Create a Reasoning Engine](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/create)
- [A2A Agent Gateway](../a2a.md)
- [Vertex AI Provider](./vertex.md)

7
docs/my-website/docs/providers/vertex_ocr.md
View file

@ -140,7 +140,7 @@ with open("document.pdf", "rb") as f:
pdf_base64 = base64.b64encode(f.read()).decode()
response = litellm.ocr(
model="vertex_ai/mistral-ocr-2505",
model="vertex_ai/mistral-ocr-2505", # This doesn't work for deepseek
document={
"type": "document_url",
"document_url": f"data:application/pdf;base64,{pdf_base64}"
@ -219,7 +219,7 @@ print(f"Cost: ${response._hidden_params.get('response_cost', 0)}")
## Important Notes
:::info URL Conversion
Vertex AI OCR endpoints don't have internet access. LiteLLM automatically converts public URLs to base64 data URIs before sending requests to Vertex AI.
Vertex AI Mistral OCR endpoints don't have internet access. LiteLLM automatically converts public URLs to base64 data URIs before sending requests to Vertex AI.
:::
:::tip Regional Availability
@ -227,11 +227,14 @@ Mistral OCR is available in multiple regions. Specify `vertex_location` to use a
- `us-central1` (default)
- `europe-west1`
- `asia-southeast1`
Deepseek OCR is only available in global region.
:::
## Supported Models
- `mistral-ocr-2505` - Latest Mistral OCR model on Vertex AI
- `deepseek-ocr-maas` - Lates Deepseek OCR model on Vertex AI
Use the Vertex AI provider prefix: `vertex_ai/<model-name>`

178
docs/my-website/docs/providers/vllm_batches.md Normal file
View file

@ -0,0 +1,178 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# vLLM - Batch + Files API
LiteLLM supports vLLM's Batch and Files API for processing large volumes of requests asynchronously.
| Feature | Supported |
|---------|-----------|
| `/v1/files` | ✅ |
| `/v1/batches` | ✅ |
| Cost Tracking | ✅ |
## Quick Start
### 1. Setup config.yaml
Define your vLLM model in `config.yaml`. LiteLLM uses the model name to route batch requests to the correct vLLM server.
```yaml
model_list:
- model_name: my-vllm-model
litellm_params:
model: hosted_vllm/meta-llama/Llama-2-7b-chat-hf
api_base: http://localhost:8000 # your vLLM server
```
### 2. Start LiteLLM Proxy
```bash
litellm --config /path/to/config.yaml
```
### 3. Create Batch File
Create a JSONL file with your batch requests:
```jsonl
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "my-vllm-model", "messages": [{"role": "user", "content": "Hello!"}]}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "my-vllm-model", "messages": [{"role": "user", "content": "How are you?"}]}}
```
### 4. Upload File & Create Batch
:::tip Model Routing
LiteLLM needs to know which model (and therefore which vLLM server) to use for batch operations. Specify the model using the `x-litellm-model` header when uploading files. LiteLLM will encode this model info into the file ID, so subsequent batch operations automatically route to the correct server.
See [Multi-Account / Model-Based Routing](../batches#multi-account--model-based-routing) for more details.
:::
<Tabs>
<TabItem value="curl" label="cURL">
**Upload File**
```bash
curl http://localhost:4000/v1/files \
-H "Authorization: Bearer sk-1234" \
-H "x-litellm-model: my-vllm-model" \
-F purpose="batch" \
-F file="@batch_requests.jsonl"
```
**Create Batch**
```bash
curl http://localhost:4000/v1/batches \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"input_file_id": "file-abc123",
"endpoint": "/v1/chat/completions",
"completion_window": "24h"
}'
```
**Check Batch Status**
```bash
curl http://localhost:4000/v1/batches/batch_abc123 \
-H "Authorization: Bearer sk-1234"
```
</TabItem>
<TabItem value="python" label="Python SDK">
```python
import litellm
import asyncio
async def run_vllm_batch():
# Upload file
file_obj = await litellm.acreate_file(
file=open("batch_requests.jsonl", "rb"),
purpose="batch",
custom_llm_provider="hosted_vllm",
)
print(f"File uploaded: {file_obj.id}")
# Create batch
batch = await litellm.acreate_batch(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=file_obj.id,
custom_llm_provider="hosted_vllm",
)
print(f"Batch created: {batch.id}")
# Poll for completion
while True:
batch_status = await litellm.aretrieve_batch(
batch_id=batch.id,
custom_llm_provider="hosted_vllm",
)
print(f"Status: {batch_status.status}")
if batch_status.status == "completed":
break
elif batch_status.status in ["failed", "cancelled"]:
raise Exception(f"Batch failed: {batch_status.status}")
await asyncio.sleep(5)
# Get results
if batch_status.output_file_id:
results = await litellm.afile_content(
file_id=batch_status.output_file_id,
custom_llm_provider="hosted_vllm",
)
print(f"Results: {results}")
asyncio.run(run_vllm_batch())
```
</TabItem>
</Tabs>
## Supported Operations
| Operation | Endpoint | Method |
|-----------|----------|--------|
| Upload file | `/v1/files` | POST |
| List files | `/v1/files` | GET |
| Retrieve file | `/v1/files/{file_id}` | GET |
| Delete file | `/v1/files/{file_id}` | DELETE |
| Get file content | `/v1/files/{file_id}/content` | GET |
| Create batch | `/v1/batches` | POST |
| List batches | `/v1/batches` | GET |
| Retrieve batch | `/v1/batches/{batch_id}` | GET |
| Cancel batch | `/v1/batches/{batch_id}/cancel` | POST |
## Environment Variables
```bash
# Set vLLM server endpoint
export HOSTED_VLLM_API_BASE="http://localhost:8000"
# Optional: API key if your vLLM server requires authentication
export HOSTED_VLLM_API_KEY="your-api-key"
```
## How Model Routing Works
When you upload a file with `x-litellm-model: my-vllm-model`, LiteLLM:
1. Encodes the model name into the returned file ID
2. Uses this encoded model info to automatically route subsequent batch operations to the correct vLLM server
3. No need to specify the model again when creating batches or retrieving results
This enables multi-tenant batch processing where different teams can use different vLLM deployments through the same LiteLLM proxy.
**Learn more:** [Multi-Account / Model-Based Routing](../batches#multi-account--model-based-routing)
## Related
- [vLLM Provider Overview](./vllm)
- [Batch API Overview](../batches)
- [Files API](../files_endpoints)

22
docs/my-website/docs/proxy/alerting.md
View file

@ -215,16 +215,16 @@ general_settings:
alerting: ["slack"]
alerting_threshold: 0.0001 # (Seconds) set an artificially low threshold for testing alerting
alert_to_webhook_url: {
"llm_exceptions": "https://hooks.slack.com/services/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",
"llm_too_slow": "https://hooks.slack.com/services/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",
"llm_requests_hanging": "https://hooks.slack.com/services/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",
"budget_alerts": "https://hooks.slack.com/services/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",
"db_exceptions": "https://hooks.slack.com/services/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",
"daily_reports": "https://hooks.slack.com/services/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",
"spend_reports": "https://hooks.slack.com/services/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",
"cooldown_deployment": "https://hooks.slack.com/services/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",
"new_model_added": "https://hooks.slack.com/services/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",
"outage_alerts": "https://hooks.slack.com/services/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",
"llm_exceptions": "example-slack-webhook-url",
"llm_too_slow": "example-slack-webhook-url",
"llm_requests_hanging": "example-slack-webhook-url",
"budget_alerts": "example-slack-webhook-url",
"db_exceptions": "example-slack-webhook-url",
"daily_reports": "example-slack-webhook-url",
"spend_reports": "example-slack-webhook-url",
"cooldown_deployment": "example-slack-webhook-url",
"new_model_added": "example-slack-webhook-url",
"outage_alerts": "example-slack-webhook-url",
}
litellm_settings:
@ -399,7 +399,7 @@ curl -X GET --location 'http://0.0.0.0:4000/health/services?service=webhook' \
{
"spend": 1, # the spend for the 'event_group'
"max_budget": 0, # the 'max_budget' set for the 'event_group'
"token": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"token": "example-api-key-123",
"user_id": "default_user_id",
"team_id": null,
"user_email": null,

12
docs/my-website/docs/proxy/config_settings.md
View file

@ -346,6 +346,7 @@ router_settings:
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Currently supported: 'router_budget_limiting', 'prompt_caching' |
| ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. |
| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) |
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
### environment variables - Reference
@ -413,6 +414,12 @@ router_settings:
| AZURE_FEDERATED_TOKEN_FILE | File path to Azure federated token
| AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY | Cost per GB per day for Azure File Search service
| AZURE_SCOPE | For EntraID Auth, Scope for Azure services, defaults to "https://cognitiveservices.azure.com/.default"
| AZURE_SENTINEL_DCR_IMMUTABLE_ID | Immutable ID of the Data Collection Rule for Azure Sentinel logging
| AZURE_SENTINEL_STREAM_NAME | Stream name for Azure Sentinel logging
| AZURE_SENTINEL_CLIENT_SECRET | Client secret for Azure Sentinel authentication
| AZURE_SENTINEL_ENDPOINT | Endpoint for Azure Sentinel logging
| AZURE_SENTINEL_TENANT_ID | Tenant ID for Azure Sentinel authentication
| AZURE_SENTINEL_CLIENT_ID | Client ID for Azure Sentinel authentication
| AZURE_KEY_VAULT_URI | URI for Azure Key Vault
| AZURE_OPERATION_POLLING_TIMEOUT | Timeout in seconds for Azure operation polling
| AZURE_STORAGE_ACCOUNT_KEY | The Azure Storage Account Key to use for Authentication to Azure Blob Storage logging
@ -541,6 +548,8 @@ router_settings:
| DOCS_TITLE | Title of the documentation pages
| DOCS_URL | The path to the Swagger API documentation. **By default this is "/"**
| EMAIL_LOGO_URL | URL for the logo used in emails
| EMAIL_BUDGET_ALERT_TTL | Time-to-live for email budget alerts in seconds
| EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE | Maximum spend percentage for triggering email budget alerts
| EMAIL_SUPPORT_CONTACT | Support contact email address
| EMAIL_SIGNATURE | Custom HTML footer/signature for all emails. Can include HTML tags for formatting and links.
| EMAIL_SUBJECT_INVITATION | Custom subject template for invitation emails.
@ -596,6 +605,8 @@ router_settings:
| GREENSCALE_ENDPOINT | Endpoint URL for Greenscale service
| GRAYSWAN_API_BASE | Base URL for GraySwan API. Default is https://api.grayswan.ai
| GRAYSWAN_API_KEY | API key for GraySwan Cygnal service
| GRAYSWAN_REASONING_MODE | Reasoning mode for GraySwan guardrail
| GRAYSWAN_VIOLATION_THRESHOLD | Violation threshold for GraySwan guardrail
| GOOGLE_APPLICATION_CREDENTIALS | Path to Google Cloud credentials JSON file
| GOOGLE_CLIENT_ID | Client ID for Google OAuth
| GOOGLE_CLIENT_SECRET | Client secret for Google OAuth
@ -825,6 +836,7 @@ router_settings:
| SMTP_TLS | Flag to enable or disable TLS for SMTP connections
| SMTP_USERNAME | Username for SMTP authentication (do not set if SMTP does not require auth)
| SENDGRID_API_KEY | API key for SendGrid email service
| RESEND_API_KEY | API key for Resend email service
| SENDGRID_SENDER_EMAIL | Email address used as the sender in SendGrid email transactions
| SPEND_LOGS_URL | URL for retrieving spend logs
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000

4
docs/my-website/docs/proxy/configs.md
View file

@ -655,7 +655,7 @@ docker run --name litellm-proxy \
-e LITELLM_CONFIG_BUCKET_OBJECT_KEY="<object_key>> \
-e LITELLM_CONFIG_BUCKET_TYPE="gcs" \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-latest --detailed_debug
docker.litellm.ai/berriai/litellm-database:main-latest --detailed_debug
```
</TabItem>
@ -676,7 +676,7 @@ docker run --name litellm-proxy \
-e LITELLM_CONFIG_BUCKET_NAME=<bucket_name> \
-e LITELLM_CONFIG_BUCKET_OBJECT_KEY="<object_key>> \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-latest
docker.litellm.ai/berriai/litellm-database:main-latest
```
</TabItem>
</Tabs>

6
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -722,7 +722,7 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
```shell
[
{
"api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"api_key": "example-api-key-123",
"total_cost": 0.3201286305151999,
"total_input_tokens": 36.0,
"total_output_tokens": 1593.0,
@ -766,7 +766,7 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
```shell
[
{
"api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"api_key": "example-api-key-123",
"total_cost": 0.00013132,
"total_input_tokens": 105.0,
"total_output_tokens": 872.0,
@ -1151,7 +1151,7 @@ curl -X GET "http://0.0.0.0:4000/spend/logs?request_id=<your-call-id" \ # e.g.:
"request_id": "chatcmpl-9ZKMURhVYSi9D6r6PJ9vLcayIK0Vm",
"call_type": "acompletion",
"metadata": {
"user_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"user_api_key": "example-api-key-123",
"user_api_key_alias": null,
"spend_logs_metadata": { # 👈 LOGGED CUSTOM METADATA
"hello": "world"

27
docs/my-website/docs/proxy/custom_auth.md
View file

@ -9,6 +9,7 @@ You can now override the default api key auth.
Make sure the response type follows the `UserAPIKeyAuth` pydantic object. This is used by for logging usage specific to that user key.
```python
from fastapi import Request
from litellm.proxy._types import UserAPIKeyAuth
async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
@ -114,6 +115,29 @@ UserAPIKeyAuth(
)
```
### Object Permission Example (MCP, agents, etc.)
```python
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
def _server_id(name: str) -> str:
server = global_mcp_server_manager.get_mcp_server_by_name(name)
if not server:
raise ValueError(f"Unknown MCP server '{name}'")
return server.server_id
object_permission = LiteLLM_ObjectPermissionTable(
mcp_servers=[_server_id("deepwiki"), _server_id("everything")], # MCP servers this key is allowed to use
mcp_tool_permissions={"deepwiki": ["search", "read_doc"]}, # optional per-server tool allow-list
)
UserAPIKeyAuth(
object_permission=object_permission,
)
```
### Advanced Configuration
```python
UserAPIKeyAuth(
@ -139,6 +163,7 @@ UserAPIKeyAuth(
### Complete Example
```python
from fastapi import Request
from datetime import datetime, timedelta
from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles
@ -333,4 +358,4 @@ async def user_api_key_auth(
except Exception:
raise Exception("Invalid API key")
```
```

2
docs/my-website/docs/proxy/customers.md
View file

@ -103,7 +103,7 @@ Expected Response
{
"spend": 0.0011120000000000001, # 👈 SPEND
"max_budget": null,
"token": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"token": "example-api-key-123",
"customer_id": "krrish12", # 👈 CUSTOMER ID
"user_id": null,
"team_id": null,

82
docs/my-website/docs/proxy/deploy.md
View file

@ -10,10 +10,38 @@ You can find the Dockerfile to build litellm proxy [here](https://github.com/Ber
## Quick Start
:::info
Facing issues with pulling the docker image? Email us at support@berri.ai.
:::
To start using Litellm, run the following commands in a shell:
<Tabs>
<TabItem value="docker" label="Docker">
```
docker pull docker.litellm.ai/berriai/litellm:main-latest
```
[**See all docker images**](https://github.com/orgs/BerriAI/packages)
</TabItem>
<TabItem value="pip" label="LiteLLM CLI (pip package)">
```shell
$ pip install 'litellm[proxy]'
```
</TabItem>
<TabItem value="docker-compose" label="Docker Compose (Proxy + DB)">
Use this docker compose to spin up the proxy with a postgres database running locally.
```bash
# Get the code
# Get the docker compose file
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/prometheus.yml
@ -30,6 +58,8 @@ echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
docker compose up
```
</TabItem>
</Tabs>
### Docker Run
@ -57,7 +87,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-stable \
docker.litellm.ai/berriai/litellm:main-stable \
--config /app/config.yaml --detailed_debug
```
@ -87,12 +117,12 @@ See all supported CLI args [here](https://docs.litellm.ai/docs/proxy/cli):
Here's how you can run the docker image and pass your config to `litellm`
```shell
docker run ghcr.io/berriai/litellm:main-stable --config your_config.yaml
docker run docker.litellm.ai/berriai/litellm:main-stable --config your_config.yaml
```
Here's how you can run the docker image and start litellm on port 8002 with `num_workers=8`
```shell
docker run ghcr.io/berriai/litellm:main-stable --port 8002 --num_workers 8
docker run docker.litellm.ai/berriai/litellm:main-stable --port 8002 --num_workers 8
```
@ -100,7 +130,7 @@ docker run ghcr.io/berriai/litellm:main-stable --port 8002 --num_workers 8
```shell
# Use the provided base image
FROM ghcr.io/berriai/litellm:main-stable
FROM docker.litellm.ai/berriai/litellm:main-stable
# Set the working directory to /app
WORKDIR /app
@ -242,7 +272,7 @@ spec:
spec:
containers:
- name: litellm
image: ghcr.io/berriai/litellm:main-stable # it is recommended to fix a version generally
image: docker.litellm.ai/berriai/litellm:main-stable # it is recommended to fix a version generally
args:
- "--config"
- "/app/proxy_server_config.yaml"
@ -279,9 +309,9 @@ Use this when you want to use litellm helm chart as a dependency for other chart
#### Step 1. Pull the litellm helm chart
```bash
helm pull oci://ghcr.io/berriai/litellm-helm
helm pull oci://docker.litellm.ai/berriai/litellm-helm
# Pulled: ghcr.io/berriai/litellm-helm:0.1.2
# Pulled: docker.litellm.ai/berriai/litellm-helm:0.1.2
# Digest: sha256:7d3ded1c99c1597f9ad4dc49d84327cf1db6e0faa0eeea0c614be5526ae94e2a
```
@ -340,7 +370,7 @@ Requirements:
We maintain a [separate Dockerfile](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) for reducing build time when running LiteLLM proxy with a connected Postgres Database
```shell
docker pull ghcr.io/berriai/litellm-database:main-stable
docker pull docker.litellm.ai/berriai/litellm-database:main-stable
```
```shell
@ -351,7 +381,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-stable \
docker.litellm.ai/berriai/litellm-database:main-stable \
--config /app/config.yaml --detailed_debug
```
@ -379,7 +409,7 @@ spec:
spec:
containers:
- name: litellm-container
image: ghcr.io/berriai/litellm:main-stable
image: docker.litellm.ai/berriai/litellm:main-stable
imagePullPolicy: Always
env:
- name: AZURE_API_KEY
@ -516,9 +546,9 @@ Use this when you want to use litellm helm chart as a dependency for other chart
#### Step 1. Pull the litellm helm chart
```bash
helm pull oci://ghcr.io/berriai/litellm-helm
helm pull oci://docker.litellm.ai/berriai/litellm-helm
# Pulled: ghcr.io/berriai/litellm-helm:0.1.2
# Pulled: docker.litellm.ai/berriai/litellm-helm:0.1.2
# Digest: sha256:7d3ded1c99c1597f9ad4dc49d84327cf1db6e0faa0eeea0c614be5526ae94e2a
```
@ -575,7 +605,7 @@ router_settings:
Start docker container with config
```shell
docker run ghcr.io/berriai/litellm:main-stable --config your_config.yaml
docker run docker.litellm.ai/berriai/litellm:main-stable --config your_config.yaml
```
### Deploy with Database + Redis
@ -610,7 +640,7 @@ Start `litellm-database`docker container with config
docker run --name litellm-proxy \
-e DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname> \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-stable --config your_config.yaml
docker.litellm.ai/berriai/litellm-database:main-stable --config your_config.yaml
```
### (Non Root) - without Internet Connection
@ -620,7 +650,7 @@ By default `prisma generate` downloads [prisma's engine binaries](https://www.pr
Use this docker image to deploy litellm with pre-generated prisma binaries.
```bash
docker pull ghcr.io/berriai/litellm-non_root:main-stable
docker pull docker.litellm.ai/berriai/litellm-non_root:main-stable
```
[Published Docker Image link](https://github.com/BerriAI/litellm/pkgs/container/litellm-non_root)
@ -639,7 +669,7 @@ Use this, If you need to set ssl certificates for your on prem litellm proxy
Pass `ssl_keyfile_path` (Path to the SSL keyfile) and `ssl_certfile_path` (Path to the SSL certfile) when starting litellm proxy
```shell
docker run ghcr.io/berriai/litellm:main-stable \
docker run docker.litellm.ai/berriai/litellm:main-stable \
--ssl_keyfile_path ssl_test/keyfile.key \
--ssl_certfile_path ssl_test/certfile.crt
```
@ -654,7 +684,7 @@ Step 1. Build your custom docker image with hypercorn
```shell
# Use the provided base image
FROM ghcr.io/berriai/litellm:main-stable
FROM docker.litellm.ai/berriai/litellm:main-stable
# Set the working directory to /app
WORKDIR /app
@ -702,7 +732,7 @@ Usage Example:
In this example, we set the keepalive timeout to 75 seconds.
```shell showLineNumbers title="docker run"
docker run ghcr.io/berriai/litellm:main-stable \
docker run docker.litellm.ai/berriai/litellm:main-stable \
--keepalive_timeout 75
```
@ -711,7 +741,7 @@ In this example, we set the keepalive timeout to 75 seconds.
```shell showLineNumbers title="Environment Variable"
export KEEPALIVE_TIMEOUT=75
docker run ghcr.io/berriai/litellm:main-stable
docker run docker.litellm.ai/berriai/litellm:main-stable
```
@ -722,7 +752,7 @@ Use this to mitigate memory growth by recycling workers after a fixed number of
Usage Examples:
```shell showLineNumbers title="docker run (CLI flag)"
docker run ghcr.io/berriai/litellm:main-stable \
docker run docker.litellm.ai/berriai/litellm:main-stable \
--max_requests_before_restart 10000
```
@ -730,7 +760,7 @@ Or set via environment variable:
```shell showLineNumbers title="Environment Variable"
export MAX_REQUESTS_BEFORE_RESTART=10000
docker run ghcr.io/berriai/litellm:main-stable
docker run docker.litellm.ai/berriai/litellm:main-stable
```
@ -759,7 +789,7 @@ docker run --name litellm-proxy \
-e LITELLM_CONFIG_BUCKET_OBJECT_KEY="<object_key>> \
-e LITELLM_CONFIG_BUCKET_TYPE="gcs" \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-stable --detailed_debug
docker.litellm.ai/berriai/litellm-database:main-stable --detailed_debug
```
</TabItem>
@ -780,7 +810,7 @@ docker run --name litellm-proxy \
-e LITELLM_CONFIG_BUCKET_NAME=<bucket_name> \
-e LITELLM_CONFIG_BUCKET_OBJECT_KEY="<object_key>> \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-stable
docker.litellm.ai/berriai/litellm-database:main-stable
```
</TabItem>
</Tabs>
@ -907,7 +937,7 @@ Run the following command, replacing `<database_url>` with the value you copied
docker run --name litellm-proxy \
-e DATABASE_URL=<database_url> \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-stable
docker.litellm.ai/berriai/litellm-database:main-stable
```
#### 4. Access the Application:
@ -986,7 +1016,7 @@ services:
context: .
args:
target: runtime
image: ghcr.io/berriai/litellm:main-stable
image: docker.litellm.ai/berriai/litellm:main-stable
ports:
- "4000:4000" # Map the container port to the host, change the host port if necessary
volumes:

6
docs/my-website/docs/proxy/docker_quick_start.md
View file

@ -20,7 +20,7 @@ End-to-End tutorial for LiteLLM Proxy to:
<TabItem value="docker" label="Docker">
```
docker pull ghcr.io/berriai/litellm:main-latest
docker pull docker.litellm.ai/berriai/litellm:main-latest
```
[**See all docker images**](https://github.com/orgs/BerriAI/packages)
@ -119,7 +119,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml --detailed_debug
# RUNNING on http://0.0.0.0:4000
@ -302,7 +302,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml --detailed_debug
```

2
docs/my-website/docs/proxy/enterprise.md
View file

@ -29,7 +29,7 @@ Features:
- **Spend Tracking & Data Exports**
- ✅ [Set USD Budgets Spend for Custom Tags](./provider_budget_routing#-tag-budgets)
- ✅ [Set Model budgets for Virtual Keys](./users#-virtual-key-model-specific)
- ✅ [Exporting LLM Logs to GCS Bucket, Azure Blob Storage](./proxy/bucket#🪣-logging-gcs-s3-buckets)
- ✅ [Exporting LLM Logs to GCS Bucket, Azure Blob Storage](../observability/gcs_bucket_integration)
- ✅ [`/spend/report` API endpoint](cost_tracking.md#✨-enterprise-api-endpoints-to-get-spend)
- **Control Guardrails per API Key/Team**
- **Custom Branding**

351
docs/my-website/docs/proxy/guardrails/guardrail_load_balancing.md Normal file
View file

@ -0,0 +1,351 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Guardrail Load Balancing
Load balance guardrail requests across multiple guardrail deployments. This is useful when you have rate limits on guardrail providers (e.g., AWS Bedrock Guardrails) and want to distribute requests across multiple accounts or regions.
## How It Works
```mermaid
flowchart LR
subgraph LiteLLM Gateway
Router[Router]
G1[Guardrail Instance A]
G2[Guardrail Instance B]
G3[Guardrail Instance N]
end
Client[Client Request] --> Router
Router -->|Round Robin / Weighted| G1
Router -->|Round Robin / Weighted| G2
Router -->|Round Robin / Weighted| G3
G1 --> AWS1[AWS Account 1]
G2 --> AWS2[AWS Account 2]
G3 --> AWSN[AWS Account N]
```
When you define multiple guardrails with the **same `guardrail_name`**, LiteLLM automatically load balances requests across them using the router's load balancing strategy.
## Why Use Guardrail Load Balancing?
| Use Case | Benefit |
|----------|---------|
| **AWS Bedrock Rate Limits** | Bedrock Guardrails have per-account rate limits. Distribute across multiple AWS accounts to increase throughput |
| **Multi-Region Redundancy** | Deploy guardrails across regions for failover and lower latency |
| **Cost Optimization** | Spread usage across accounts with different pricing tiers or credits |
| **A/B Testing** | Test different guardrail configurations with weighted distribution |
## Quick Start
### 1. Define Multiple Guardrails with Same Name
Define multiple guardrail entries with the **same `guardrail_name`** but different configurations:
<Tabs>
<TabItem value="bedrock" label="Bedrock Guardrails">
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
guardrails:
# First Bedrock guardrail - AWS Account 1
- guardrail_name: "content-filter"
litellm_params:
guardrail: bedrock/guardrail
mode: "pre_call"
guardrailIdentifier: "abc123"
guardrailVersion: "1"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID_1
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY_1
aws_region_name: "us-east-1"
# Second Bedrock guardrail - AWS Account 2
- guardrail_name: "content-filter"
litellm_params:
guardrail: bedrock/guardrail
mode: "pre_call"
guardrailIdentifier: "def456"
guardrailVersion: "1"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID_2
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY_2
aws_region_name: "us-west-2"
```
</TabItem>
<TabItem value="custom" label="Custom Guardrails">
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
guardrails:
# First custom guardrail instance
- guardrail_name: "pii-filter"
litellm_params:
guardrail: custom_guardrail.PIIFilterA
mode: "pre_call"
# Second custom guardrail instance
- guardrail_name: "pii-filter"
litellm_params:
guardrail: custom_guardrail.PIIFilterB
mode: "pre_call"
```
</TabItem>
<TabItem value="aporia" label="Aporia Guardrails">
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
guardrails:
# First Aporia instance
- guardrail_name: "toxicity-filter"
litellm_params:
guardrail: aporia
mode: "pre_call"
api_key: os.environ/APORIA_API_KEY_1
api_base: os.environ/APORIA_API_BASE_1
# Second Aporia instance
- guardrail_name: "toxicity-filter"
litellm_params:
guardrail: aporia
mode: "pre_call"
api_key: os.environ/APORIA_API_KEY_2
api_base: os.environ/APORIA_API_BASE_2
```
</TabItem>
</Tabs>
### 2. Start LiteLLM Gateway
```bash showLineNumbers title="Start proxy"
litellm --config config.yaml --detailed_debug
```
### 3. Make Requests
Requests using the guardrail will be automatically load balanced:
```bash showLineNumbers title="Test request"
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello, how are you?"}],
"guardrails": ["content-filter"]
}'
```
## Weighted Load Balancing
Assign weights to distribute traffic unevenly across guardrail instances:
```yaml showLineNumbers title="config.yaml - Weighted distribution"
guardrails:
# 80% of traffic
- guardrail_name: "content-filter"
litellm_params:
guardrail: bedrock/guardrail
mode: "pre_call"
guardrailIdentifier: "primary-guard"
guardrailVersion: "1"
weight: 8 # Higher weight = more traffic
# 20% of traffic
- guardrail_name: "content-filter"
litellm_params:
guardrail: bedrock/guardrail
mode: "pre_call"
guardrailIdentifier: "secondary-guard"
guardrailVersion: "1"
weight: 2 # Lower weight = less traffic
```
## Bedrock Guardrails - Multi-Account Setup
AWS Bedrock Guardrails have rate limits per account. Here's how to set up load balancing across multiple AWS accounts:
### Architecture
```mermaid
flowchart TB
subgraph LiteLLM["LiteLLM Gateway"]
LB[Load Balancer]
end
subgraph AWS1["AWS Account 1 (us-east-1)"]
BG1[Bedrock Guardrail]
end
subgraph AWS2["AWS Account 2 (us-west-2)"]
BG2[Bedrock Guardrail]
end
subgraph AWS3["AWS Account 3 (eu-west-1)"]
BG3[Bedrock Guardrail]
end
Client[Client] --> LiteLLM
LB --> BG1
LB --> BG2
LB --> BG3
```
### Configuration
```yaml showLineNumbers title="config.yaml - Multi-account Bedrock"
model_list:
- model_name: claude-3
litellm_params:
model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
guardrails:
# AWS Account 1 - US East
- guardrail_name: "bedrock-content-filter"
litellm_params:
guardrail: bedrock/guardrail
mode: "during_call"
guardrailIdentifier: "guard-us-east"
guardrailVersion: "DRAFT"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_1
aws_secret_access_key: os.environ/AWS_SECRET_KEY_1
aws_region_name: "us-east-1"
# AWS Account 2 - US West
- guardrail_name: "bedrock-content-filter"
litellm_params:
guardrail: bedrock/guardrail
mode: "during_call"
guardrailIdentifier: "guard-us-west"
guardrailVersion: "DRAFT"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_2
aws_secret_access_key: os.environ/AWS_SECRET_KEY_2
aws_region_name: "us-west-2"
# AWS Account 3 - EU West
- guardrail_name: "bedrock-content-filter"
litellm_params:
guardrail: bedrock/guardrail
mode: "during_call"
guardrailIdentifier: "guard-eu-west"
guardrailVersion: "DRAFT"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_3
aws_secret_access_key: os.environ/AWS_SECRET_KEY_3
aws_region_name: "eu-west-1"
```
### Test Multi-Account Setup
```bash showLineNumbers title="Run multiple requests to verify load balancing"
# Run 10 requests - they will be distributed across accounts
for i in {1..10}; do
curl -s -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "claude-3",
"messages": [{"role": "user", "content": "Hello"}],
"guardrails": ["bedrock-content-filter"]
}' &
done
wait
```
Check proxy logs to verify requests are distributed across different AWS accounts.
## Custom Guardrails Example
Create two custom guardrail classes for load balancing:
```python showLineNumbers title="custom_guardrail.py"
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy._types import UserAPIKeyAuth
from litellm.caching.caching import DualCache
class PIIFilterA(CustomGuardrail):
"""PII Filter Instance A"""
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: str,
):
print("PIIFilterA processing request")
# Your PII filtering logic here
return data
class PIIFilterB(CustomGuardrail):
"""PII Filter Instance B"""
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: str,
):
print("PIIFilterB processing request")
# Your PII filtering logic here
return data
```
```yaml showLineNumbers title="config.yaml"
guardrails:
- guardrail_name: "pii-filter"
litellm_params:
guardrail: custom_guardrail.PIIFilterA
mode: "pre_call"
- guardrail_name: "pii-filter"
litellm_params:
guardrail: custom_guardrail.PIIFilterB
mode: "pre_call"
```
## Verifying Load Balancing
Enable detailed debug logging to verify load balancing is working:
```bash showLineNumbers title="Start with debug logging"
litellm --config config.yaml --detailed_debug
```
You should see logs indicating which guardrail instance is selected:
```
Selected guardrail deployment: bedrock/guardrail (guard-us-east)
Selected guardrail deployment: bedrock/guardrail (guard-us-west)
Selected guardrail deployment: bedrock/guardrail (guard-eu-west)
...
```
## Related
- [Guardrails Quick Start](./quick_start.md)
- [Bedrock Guardrails](./bedrock.md)
- [Custom Guardrails](./custom_guardrail.md)
- [Load Balancing for LLM Calls](../load_balancing.md)

11
docs/my-website/docs/proxy/guardrails/lakera_ai.md
View file

@ -29,6 +29,13 @@ guardrails:
mode: "pre_call"
api_key: os.environ/LAKERA_API_KEY
api_base: os.environ/LAKERA_API_BASE
- guardrail_name: "lakera-monitor"
litellm_params:
guardrail: lakera_v2
mode: "pre_call"
on_flagged: "monitor" # Log violations but don't block
api_key: os.environ/LAKERA_API_KEY
api_base: os.environ/LAKERA_API_BASE
```
@ -144,6 +151,7 @@ guardrails:
# breakdown: Optional[bool] = True,
# metadata: Optional[Dict] = None,
# dev_info: Optional[bool] = True,
# on_flagged: Optional[str] = "block", # "block" or "monitor"
```
- `api_base`: (Optional[str]) The base of the Lakera integration. Defaults to `https://api.lakera.ai`
@ -153,3 +161,6 @@ guardrails:
- `breakdown`: (Optional[bool]) When true the response will return a breakdown list of the detectors that were run, as defined in the policy, and whether each of them detected something or not.
- `metadata`: (Optional[Dict]) Metadata tags can be attached to screening requests as an object that can contain any arbitrary key-value pairs.
- `dev_info`: (Optional[bool]) When true the response will return an object with developer information about the build of Lakera Guard.
- `on_flagged`: (Optional[str]) Action to take when content is flagged. Defaults to `"block"`.
- `"block"`: Raises an HTTP 400 exception when violations are detected (default behavior)
- `"monitor"`: Logs violations but allows the request to proceed. Useful for tuning security policies without blocking legitimate requests.

415
docs/my-website/docs/proxy/guardrails/litellm_content_filter.md
View file

@ -3,10 +3,12 @@ import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# LiteLLM Content Filter
# LiteLLM Content Filter (Built-in Guardrails)
**Built-in guardrail** for detecting and filtering sensitive information using regex patterns and keyword matching. No external dependencies required.
**When to use?** Good for cases which do not require an ML model to detect sensitive information.
## Overview
| Property | Details |
@ -56,6 +58,44 @@ Test examples:
### Step 1: Define Guardrails in config.yaml
<Tabs>
<TabItem label="Harmful Content Detection" value="harmful">
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "harmful-content-filter"
litellm_params:
guardrail: litellm_content_filter
mode: "pre_call"
# Enable harmful content categories
categories:
- category: "harmful_self_harm"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
- category: "harmful_violence"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
- category: "harmful_illegal_weapons"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
```
</TabItem>
<TabItem label="PII Protection" value="pii">
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-3.5-turbo
@ -86,6 +126,48 @@ guardrails:
description: "Sensitive internal information"
```
</TabItem>
<TabItem label="Combined" value="combined">
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "comprehensive-filter"
litellm_params:
guardrail: litellm_content_filter
mode: "pre_call"
# Harmful content categories
categories:
- category: "harmful_violence"
enabled: true
action: "BLOCK"
severity_threshold: "high"
# PII patterns
patterns:
- pattern_type: "prebuilt"
pattern_name: "us_ssn"
action: "BLOCK"
- pattern_type: "prebuilt"
pattern_name: "email"
action: "MASK"
# Custom keywords
blocked_words:
- keyword: "confidential"
action: "BLOCK"
```
</TabItem>
</Tabs>
### Step 2: Start LiteLLM Gateway
```shell
@ -175,7 +257,7 @@ Contact me at [EMAIL_REDACTED]
| `amex` | American Express cards | `3782-822463-10005` |
| `aws_access_key` | AWS access keys | `AKIAIOSFODNN7EXAMPLE` |
| `aws_secret_key` | AWS secret keys | `wJalrXUtnFEMI/K7MDENG/bPxRfi...` |
| `github_token` | GitHub tokens | `ghp_16C7e42F292c6912E7710c838347Ae178B4a` |
| `github_token` | GitHub tokens | `example-github-token-123` |
### Using Prebuilt Patterns
@ -310,6 +392,85 @@ for chunk in response:
# Emails automatically masked in real-time
```
## Image Content Filtering
Content filter can analyze images by generating descriptions and applying filters to the text descriptions.
:::warning
This can introduce significant latency to the request - depending on the speed of the vision-capable model.
This is because, each request containing images will be sent to the vision-capable model to generate a description.
:::
### Configuration
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4-vision
litellm_params:
model: openai/gpt-4-vision-preview
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "image-filter"
litellm_params:
guardrail: litellm_content_filter
mode: "pre_call"
image_model: "gpt-4-vision" # value is `model_name` of the vision-capable model
# Apply same filters to image descriptions
categories:
- category: "harmful_violence"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
patterns:
- pattern_type: "prebuilt"
pattern_name: "email"
action: "MASK"
```
### How It Works
1. Image is sent to the vision model to generate a text description
2. Content filters are applied to the description
3. If harmful content is detected, request is blocked with context about the image
**Example:**
```python
import openai
client = openai.OpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
response = client.chat.completions.create(
model="gpt-4-vision",
messages=[{
"role": "user",
"content": [
{"type": "text", "text": "What's in this image?"},
{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}
]
}],
extra_body={"guardrails": ["image-filter"]}
)
```
If the image description contains filtered content, you'll get:
```json
{
"error": "Content blocked: harmful_violence category keyword 'weapon' detected (severity: high) (Image description): The image shows..."
}
```
## Customizing Redaction Tags
When using the `MASK` action, sensitive content is replaced with redaction tags. You can customize how these tags appear.
@ -363,9 +524,171 @@ Output: "Email ***EMAIL***, SSN ***US_SSN***, ***REDACTED*** data"
- Pattern names are automatically uppercased (e.g., `email` → `EMAIL`)
- `keyword_redaction_tag` is a fixed string (no placeholders)
## Content Categories
Prebuilt categories use **keyword matching** to detect harmful content, bias, and inappropriate advice. Keywords are matched with word boundaries (single words) or as substrings (multi-word phrases), case-insensitive.
### Available Categories
| Category | Description |
|----------|-------------|
| **Harmful Content** | |
| `harmful_self_harm` | Self-harm, suicide, eating disorders |
| `harmful_violence` | Violence, criminal planning, attacks |
| `harmful_illegal_weapons` | Illegal weapons, explosives, dangerous materials |
| **Bias Detection** | |
| `bias_gender` | Gender-based discrimination, stereotypes |
| `bias_sexual_orientation` | LGBTQ+ discrimination, homophobia, transphobia |
| `bias_racial` | Racial/ethnic discrimination, stereotypes |
| `bias_religious` | Religious discrimination, stereotypes |
| **Denied Advice** | |
| `denied_financial_advice` | Personalized financial advice, investment recommendations |
| `denied_medical_advice` | Medical advice, diagnosis, treatment recommendations |
| `denied_legal_advice` | Legal advice, representation, legal strategy |
:::info Bias Detection Considerations
Bias detection is **complex and context-dependent**. Rule-based systems catch explicit discriminatory language but may generate false positives on legitimate discussions. Start with **high severity thresholds** and test thoroughly. For mission-critical bias detection, consider combining with AI-based guardrails (e.g., HiddenLayer, Lakera).
:::
### Configuration
```yaml showLineNumbers title="config.yaml"
guardrails:
- guardrail_name: "content-filter"
litellm_params:
guardrail: litellm_content_filter
mode: "pre_call"
categories:
- category: "harmful_self_harm"
enabled: true
action: "BLOCK"
severity_threshold: "medium" # Blocks medium+ severity
- category: "bias_gender"
enabled: true
action: "BLOCK"
severity_threshold: "high" # Only explicit discrimination
- category: "denied_financial_advice"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
```
**Severity Thresholds:**
- `"high"` - Only blocks high severity items
- `"medium"` - Blocks medium and high severity (default)
- `"low"` - Blocks all severity levels
### Custom Category Files
Override default categories with custom keyword lists:
```yaml showLineNumbers title="config.yaml"
categories:
- category: "harmful_self_harm"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
category_file: "/path/to/custom.yaml"
```
```yaml showLineNumbers title="custom.yaml"
category_name: "harmful_self_harm"
description: "Custom self-harm detection"
default_action: "BLOCK"
keywords:
- keyword: "suicide"
severity: "high"
- keyword: "harm myself"
severity: "high"
exceptions:
- "suicide prevention"
- "mental health"
```
## Use Cases
### 1. PII Protection
### 1. Harmful Content Detection
Block or detect requests containing harmful, illegal, or dangerous content:
```yaml
categories:
- category: "harmful_self_harm"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
- category: "harmful_violence"
enabled: true
action: "BLOCK"
severity_threshold: "high"
- category: "harmful_illegal_weapons"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
```
### 2. Bias and Discrimination Detection
Detect and block biased, discriminatory, or hateful content across multiple dimensions:
```yaml
categories:
# Gender-based discrimination
- category: "bias_gender"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
# LGBTQ+ discrimination
- category: "bias_sexual_orientation"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
# Racial/ethnic discrimination
- category: "bias_racial"
enabled: true
action: "BLOCK"
severity_threshold: "high" # Only explicit to reduce false positives
# Religious discrimination
- category: "bias_religious"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
```
**Sensitivity Tuning:**
For bias detection, severity thresholds are critical to balance safety and legitimate discourse:
```yaml
# Conservative (low false positives, may miss subtle bias)
categories:
- category: "bias_racial"
severity_threshold: "high" # Only blocks explicit discriminatory language
# Balanced (recommended)
categories:
- category: "bias_gender"
severity_threshold: "medium" # Blocks stereotypes and explicit discrimination
# Strict (high safety, may have more false positives)
categories:
- category: "bias_sexual_orientation"
severity_threshold: "low" # Blocks all potentially problematic content
```
### 3. PII Protection
Block or mask personally identifiable information before sending to LLMs:
```yaml
@ -409,10 +732,64 @@ For large lists of sensitive terms, use a file:
blocked_words_file: "/path/to/sensitive_terms.yaml"
```
### 4. Compliance
### 4. Safe AI for Consumer Applications
Combining harmful content and bias detection for consumer-facing AI:
```yaml
guardrails:
- guardrail_name: "safe-consumer-ai"
litellm_params:
guardrail: litellm_content_filter
mode: "pre_call"
categories:
# Harmful content - strict
- category: "harmful_self_harm"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
- category: "harmful_violence"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
# Bias detection - balanced
- category: "bias_gender"
enabled: true
action: "BLOCK"
severity_threshold: "high" # Avoid blocking legitimate gender discussions
- category: "bias_sexual_orientation"
enabled: true
action: "BLOCK"
severity_threshold: "medium"
- category: "bias_racial"
enabled: true
action: "BLOCK"
severity_threshold: "high" # Education and news may discuss race
```
**Perfect for:**
- Chatbots and virtual assistants
- Educational AI tools
- Customer service AI
- Content generation platforms
- Public-facing AI applications
### 5. Compliance
Ensure regulatory compliance by filtering sensitive data types:
```yaml
# Categories checked first (high priority)
# Category keywords are matched first
categories:
- category: "harmful_self_harm"
severity_threshold: "high"
# Then regex patterns
patterns:
- pattern_type: "prebuilt"
pattern_name: "visa"
@ -422,34 +799,4 @@ patterns:
action: "BLOCK"
```
## Troubleshooting
### Pattern Not Matching
**Issue:** Regex pattern isn't detecting expected content
**Solution:** Test your regex pattern:
```python
import re
pattern = r'\b[A-Z]{3}-\d{4}\b'
test_text = "Employee ID: ABC-1234"
print(re.search(pattern, test_text)) # Should match
```
### Multiple Pattern Matches
**Issue:** Text contains multiple sensitive patterns
**Solution:** First matching pattern/keyword is processed. Order patterns by priority:
```yaml
patterns:
# Most critical first
- pattern_type: "prebuilt"
pattern_name: "us_ssn"
action: "BLOCK"
# Less critical
- pattern_type: "prebuilt"
pattern_name: "email"
action: "MASK"
```

2
docs/my-website/docs/proxy/guardrails/pangea.md
View file

@ -67,7 +67,7 @@ docker run --rm \
-e PANGEA_AI_GUARD_TOKEN=$PANGEA_AI_GUARD_TOKEN \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml
```

104
docs/my-website/docs/proxy/guardrails/pillar_security.md
View file

@ -72,13 +72,15 @@ litellm --config config.yaml --port 4000
### Overview
Pillar Security supports three execution modes for comprehensive protection:
Pillar Security supports five execution modes for comprehensive protection:
| Mode | When It Runs | What It Protects | Use Case
|------|-------------|------------------|----------
| **`pre_call`** | Before LLM call | User input only | Block malicious prompts, prevent prompt injection
| **`during_call`** | Parallel with LLM call | User input only | Input monitoring with lower latency
| **`post_call`** | After LLM response | Full conversation context | Output filtering, PII detection in responses
| **`pre_mcp_call`** | Before MCP tool call | MCP tool inputs | Validate and sanitize MCP tool call arguments
| **`during_mcp_call`** | During MCP tool call | MCP tool inputs | Real-time monitoring of MCP tool calls
### Why Dual Mode is Recommended
@ -198,6 +200,85 @@ litellm_settings:
set_verbose: true # Enable detailed logging
```
</TabItem>
<TabItem value="masking" label="Masking Mode - Auto-Sanitize PII">
**Best for:**
- 🔒 **PII Protection**: Automatically sanitize sensitive data before sending to LLM
- ✅ **Continue Workflows**: Allow requests to proceed with masked content
- 🛡️ **Zero Trust**: Never expose sensitive data to LLM models
- 📊 **Compliance**: Meet data privacy requirements without blocking legitimate requests
```yaml
model_list:
- model_name: gpt-4.1-mini
litellm_params:
model: openai/gpt-4.1-mini
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "pillar-masking"
litellm_params:
guardrail: pillar
mode: "pre_call" # Scan input before LLM call
api_key: os.environ/PILLAR_API_KEY # Your Pillar API key
api_base: os.environ/PILLAR_API_BASE # Pillar API endpoint
on_flagged_action: "mask" # Mask sensitive content instead of blocking
persist_session: true # Keep records for investigation
include_scanners: true # Understand which scanners triggered
include_evidence: true # Capture evidence for analysis
default_on: true # Enable for all requests
general_settings:
master_key: "YOUR_LITELLM_PROXY_MASTER_KEY"
litellm_settings:
set_verbose: true
```
**How it works:**
1. User sends request with sensitive data: `"My email is john@example.com"`
2. Pillar detects PII and returns masked version: `"My email is [MASKED_EMAIL]"`
3. LiteLLM replaces original messages with masked messages
4. Request proceeds to LLM with sanitized content
5. User receives response without exposing sensitive data
</TabItem>
<TabItem value="mcp" label="MCP Call Protection">
**Best for:**
- 🤖 **Agent Workflows**: Protect MCP (Model Context Protocol) tool calls
- 🔒 **Tool Input Validation**: Scan arguments passed to MCP tools
- 🛡️ **Comprehensive Coverage**: Extend security to all LLM endpoints
```yaml
model_list:
- model_name: gpt-4.1-mini
litellm_params:
model: openai/gpt-4.1-mini
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "pillar-mcp-guard"
litellm_params:
guardrail: pillar
mode: "pre_mcp_call" # Scan MCP tool call inputs
api_key: os.environ/PILLAR_API_KEY # Your Pillar API key
api_base: os.environ/PILLAR_API_BASE # Pillar API endpoint
on_flagged_action: "block" # Block malicious MCP calls
default_on: true # Enable for all MCP calls
general_settings:
master_key: "YOUR_LITELLM_PROXY_MASTER_KEY"
litellm_settings:
set_verbose: true
```
**MCP Modes:**
- `pre_mcp_call`: Scan MCP tool call inputs before execution
- `during_mcp_call`: Monitor MCP tool calls in real-time
</TabItem>
</Tabs>
@ -251,6 +332,15 @@ Logs the violation but allows the request to proceed:
on_flagged_action: "monitor"
```
#### Mask
Automatically sanitizes sensitive content (PII, secrets, etc.) in your messages before sending them to the LLM:
```yaml
on_flagged_action: "mask"
```
When masking is enabled, sensitive information is automatically replaced with masked versions, allowing requests to proceed safely without exposing sensitive data to the LLM.
**Response Headers:**
You can opt in to receiving detection details in response headers by configuring `include_scanners: true` and/or `include_evidence: true`. When enabled, these headers are included for **every request**—not just flagged ones—enabling comprehensive metrics, false positive analysis, and threat investigation.
@ -383,7 +473,8 @@ export PILLAR_TIMEOUT="5.0"
**Quick takeaways**
- Every request still runs *all* Pillar scanners; these options only change what comes back.
- Choose richer responses when you need audit trails, lighter responses when latency or cost matters.
- Blocking is controlled by LiteLLM’s `on_flagged_action` configuration—Pillar headers do not change block/monitor behaviour.
- Actions (block/monitor/mask) are controlled by LiteLLM's `on_flagged_action` configuration—Pillar headers are automatically set based on your config.
- When blocking (`on_flagged_action: "block"`), the `include_scanners` and `include_evidence` settings control what details are included in the exception response.
Pillar Security executes the full scanner suite on each call. The settings below tune the Protect response headers LiteLLM sends, letting you balance fidelity, retention, and latency.
@ -415,9 +506,10 @@ include_evidence: true # → plr_evidence (default true in LiteLLM)
```
Use when you only care about whether Pillar detected a threat.
> **📝 Note:** `flagged: true` means Pillar’s scanners recommend blocking. Pillar only reports this verdict—LiteLLM enforces your policy via the `on_flagged_action` configuration (no Pillar header controls it):
> - `on_flagged_action: "block"` → LiteLLM raises a 400 guardrail error
> **📝 Note:** `flagged: true` means Pillar's scanners recommend blocking. Pillar only reports this verdict—LiteLLM enforces your policy via the `on_flagged_action` configuration:
> - `on_flagged_action: "block"` → LiteLLM raises a 400 guardrail error (exception includes scanners/evidence based on `include_scanners`/`include_evidence` settings)
> - `on_flagged_action: "monitor"` → LiteLLM logs the threat but still returns the LLM response
> - `on_flagged_action: "mask"` → LiteLLM replaces messages with masked versions and allows the request to proceed
- **Scanner breakdown** (`include_scanners=true`)
```json
@ -698,7 +790,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \
"messages": [
{
"role": "user",
"content": "Generate python code that accesses my Github repo using this PAT: ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8"
"content": "Generate python code that accesses my Github repo using this PAT: example-github-token-123"
}
],
"max_tokens": 50
@ -723,7 +815,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \
"type": "github_token",
"start_idx": 66,
"end_idx": 106,
"evidence": "ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8",
"evidence": "example-github-token-123",
}
]
}

7
docs/my-website/docs/proxy/guardrails/quick_start.md
View file

@ -69,6 +69,13 @@ guardrails:
- `during_call` Run **during** LLM call, on **input** Same as `pre_call` but runs in parallel as LLM call. Response not returned until guardrail check completes
- A list of the above values to run multiple modes, e.g. `mode: [pre_call, post_call]`
### Load Balancing Guardrails
Need to distribute guardrail requests across multiple accounts or regions? See [Guardrail Load Balancing](./guardrail_load_balancing.md) for details on:
- Load balancing across multiple AWS Bedrock accounts (useful for rate limit management)
- Weighted distribution across guardrail instances
- Multi-region guardrail deployments
## 2. Start LiteLLM Gateway

25
docs/my-website/docs/proxy/load_balancing.md
View file

@ -29,6 +29,10 @@ LiteLLM automatically distributes requests across multiple deployments of the sa
| **latency-based-routing** | Routes to fastest responding deployment | Latency-critical applications |
| **cost-based-routing** | Routes to deployment with lowest cost | Cost-sensitive applications |
:::tip Deployment Priority
Use the `order` parameter to prioritize specific deployments. [See Deployment Ordering](#deployment-ordering-priority) for details.
:::
## Quick Start - Load Balancing
#### Step 1 - Set deployments on config
@ -243,6 +247,27 @@ class RouterModelGroupAliasItem(TypedDict):
hidden: bool # if 'True', don't return on `/v1/models`, `/v1/model/info`, `/v1/model_group/info`
```
## Deployment Ordering (Priority)
Set `order` in `litellm_params` to prioritize deployments. Lower values = higher priority. When multiple deployments share the same `order`, the routing strategy picks among them.
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: azure/gpt-4-primary
api_key: os.environ/AZURE_API_KEY
order: 1 # 👈 Highest priority - always tried first
- model_name: gpt-4
litellm_params:
model: azure/gpt-4-fallback
api_key: os.environ/AZURE_API_KEY_2
order: 2 # 👈 Used when order=1 is unavailable
```
If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments.
### When You'll See Load Balancing in Action
**Immediate Effects:**

5
docs/my-website/docs/proxy/logging.md
View file

@ -16,6 +16,7 @@ Log Proxy input, output, and exceptions using:
- Custom Callbacks - Custom code and API endpoints
- Langsmith
- DataDog
- Azure Sentinel
- DynamoDB
- etc.
@ -1574,6 +1575,10 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
👉 Go here for using [Datadog LLM Observability](../observability/datadog) with LiteLLM Proxy
## [Azure Sentinel](../observability/azure_sentinel)
👉 Go here for using [Azure Sentinel](../observability/azure_sentinel) with LiteLLM Proxy
## Lunary
#### Step1: Install dependencies and set your environment variables

2
docs/my-website/docs/proxy/multiple_admins.md
View file

@ -89,7 +89,7 @@ curl -X POST 'http://0.0.0.0:4000/team/update' \
"id": "bd136c28-edd0-4cb6-b963-f35464cf6f5a",
"updated_at": "2024-06-08 23:41:14.793",
"changed_by": "krrish@berri.ai", # 👈 CHANGED BY
"changed_by_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"changed_by_api_key": "example-api-key-123",
"action": "updated",
"table_name": "LiteLLM_TeamTable",
"object_id": "8bf18b11-7f52-4717-8e1f-7c65f9d01e52",

2
docs/my-website/docs/proxy/prod.md
View file

@ -33,7 +33,7 @@ litellm_settings:
Set slack webhook url in your env
```shell
export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/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"
export SLACK_WEBHOOK_URL="example-slack-webhook-url"
```
Turn off FASTAPI's default info logs

2
docs/my-website/docs/proxy/quick_start.md
View file

@ -400,7 +400,7 @@ from anthropic import Anthropic
client = Anthropic(
base_url="http://localhost:4000", # proxy endpoint
api_key="sk-s4xN1IiLTCytwtZFJaYQrA", # litellm proxy virtual key
api_key="sk-test-proxy-key-123", # litellm proxy virtual key (example)
)
message = client.messages.create(

2
docs/my-website/docs/proxy/shared_health_check.md
View file

@ -269,7 +269,7 @@ spec:
spec:
containers:
- name: litellm-proxy
image: ghcr.io/berriai/litellm:latest
image: docker.litellm.ai/berriai/litellm:latest
env:
- name: USE_SHARED_HEALTH_CHECK
value: "true"

2
docs/my-website/docs/proxy/user_keys.md
View file

@ -285,7 +285,7 @@ from anthropic import Anthropic
client = Anthropic(
base_url="http://localhost:4000", # proxy endpoint
api_key="sk-s4xN1IiLTCytwtZFJaYQrA", # litellm proxy virtual key
api_key="sk-test-proxy-key-123", # litellm proxy virtual key (example)
)
message = client.messages.create(

101
docs/my-website/docs/reasoning_content.md
View file

@ -114,6 +114,107 @@ curl http://0.0.0.0:4000/v1/chat/completions \
Here's how to use `thinking` blocks by Anthropic with tool calling.
### Important: OpenAI-Compatible API Limitations
:::warning Compatibility Notice
Anthropic extended thinking with tool calling is **not fully compatible** with OpenAI-compatible API clients. This is due to fundamental architectural differences between how OpenAI and Anthropic handle reasoning in multi-turn conversations.
:::
When using Anthropic models with `thinking` enabled and tool calling, you **must include `thinking_blocks`** from the previous assistant response when sending tool results back. Failure to do so will result in a `400 Bad Request` error.
**OpenAI vs Anthropic Architecture:**
| Provider | API Architecture | Reasoning Storage | Multi-turn Handling |
|----------|------------------|-------------------|---------------------|
| **OpenAI** (o1, o3) | Responses API (Stateful) | Server-side | Server stores reasoning internally; client sends `previous_response_id` |
| **Anthropic** (Claude) | Messages API (Stateless) | Client-side | Client must store and resend `thinking_blocks` with every request |
1. OpenAI's Chat Completions spec has **no field** for `thinking_blocks`
2. OpenAI-compatible clients (LibreChat, Open WebUI, Vercel AI SDK, etc.) **ignore** the `thinking_blocks` field in responses
3. When these clients reconstruct the assistant message for the next turn, the thinking blocks are lost
4. Anthropic rejects the request because the assistant message doesn't start with a thinking block
:::tip LiteLLM supports thinking_blocks
LiteLLM's `completion()` API **does support** sending `thinking_blocks` in assistant messages. If you're using LiteLLM directly (not through an OpenAI-compatible client), you can preserve and resend `thinking_blocks` and everything will work correctly.
:::
**Solutions:**
1. **Use LiteLLM's built-in workaround** (recommended): Set `litellm.modify_params = True` and LiteLLM will automatically handle this incompatibility by dropping the `thinking` param when `thinking_blocks` are missing (see below)
2. **For client developers**: Explicitly handle and resend the `thinking_blocks` field (see example below)
3. **Disable extended thinking** when using tools with OpenAI-compatible clients that don't support `thinking_blocks`
4. **Use Anthropic's native API** directly instead of OpenAI-compatible endpoints
### LiteLLM Built-in Workaround
LiteLLM can automatically handle this incompatibility when `modify_params=True` is set. If the client sends a request with `thinking` enabled but the assistant message with `tool_calls` is missing `thinking_blocks`, LiteLLM will automatically drop the `thinking` param for that turn to avoid the error.
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers
import litellm
# Enable automatic parameter modification
litellm.modify_params = True
# Now this will work even if thinking_blocks are missing from the assistant message
response = litellm.completion(
model="anthropic/claude-sonnet-4-20250514",
thinking={"type": "enabled", "budget_tokens": 1024},
tools=[...],
messages=[
{"role": "user", "content": "What's the weather in Madrid?"},
{
"role": "assistant",
"tool_calls": [{"id": "call_123", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "Madrid"}'}}]
# Note: thinking_blocks is missing here - LiteLLM will handle it
},
{"role": "tool", "tool_call_id": "call_123", "content": "22°C sunny"}
]
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml showLineNumbers title="config.yaml"
litellm_settings:
modify_params: true # Enable automatic parameter modification
model_list:
- model_name: claude-thinking
litellm_params:
model: anthropic/claude-sonnet-4-20250514
thinking:
type: enabled
budget_tokens: 1024
```
</TabItem>
</Tabs>
:::info
When `modify_params=True` and LiteLLM drops the `thinking` param, the model will **not** use extended thinking for that specific turn. The conversation will continue normally, but without reasoning for that response.
:::
**Correct way to include `thinking_blocks`:**
```python
# After receiving a response with tool_calls, include thinking_blocks when sending back:
assistant_message = {
"role": "assistant",
"content": response.choices[0].message.content,
"tool_calls": [...],
"thinking_blocks": response.choices[0].message.thinking_blocks # ← Required!
}
```
---
<Tabs>
<TabItem value="sdk" label="SDK">

2
docs/my-website/docs/response_api.md
View file

@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
# /responses
LiteLLM provides a BETA endpoint in the spec of [OpenAI's `/responses` API](https://platform.openai.com/docs/api-reference/responses)
LiteLLM provides an endpoint in the spec of [OpenAI's `/responses` API](https://platform.openai.com/docs/api-reference/responses)
Requests to /chat/completions may be bridged here automatically when the provider lacks support for that endpoint. The model’s default `mode` determines how bridging works.(see `model_prices_and_context_window`)

53
docs/my-website/docs/routing.md
View file

@ -832,6 +832,59 @@ asyncio.run(router_acompletion())
## Basic Reliability
### Deployment Ordering (Priority)
Set `order` in `litellm_params` to prioritize deployments. Lower values = higher priority. When multiple deployments share the same `order`, the routing strategy picks among them.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import Router
model_list = [
{
"model_name": "gpt-4",
"litellm_params": {
"model": "azure/gpt-4-primary",
"api_key": os.getenv("AZURE_API_KEY"),
"order": 1, # 👈 Highest priority
},
},
{
"model_name": "gpt-4",
"litellm_params": {
"model": "azure/gpt-4-fallback",
"api_key": os.getenv("AZURE_API_KEY_2"),
"order": 2, # 👈 Used when order=1 is unavailable
},
},
]
router = Router(model_list=model_list)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: azure/gpt-4-primary
api_key: os.environ/AZURE_API_KEY
order: 1 # 👈 Highest priority
- model_name: gpt-4
litellm_params:
model: azure/gpt-4-fallback
api_key: os.environ/AZURE_API_KEY_2
order: 2 # 👈 Used when order=1 is unavailable
```
</TabItem>
</Tabs>
### Weighted Deployments
Set `weight` on a deployment to pick one deployment more often than others.

5
docs/my-website/docs/search/index.md
View file

@ -2,7 +2,7 @@
| Feature | Supported |
|---------|-----------|
| Supported Providers | `perplexity`, `tavily`, `parallel_ai`, `exa_ai`, `google_pse`, `dataforseo`, `firecrawl`, `searxng` |
| Supported Providers | `perplexity`, `tavily`, `parallel_ai`, `exa_ai`, `google_pse`, `dataforseo`, `firecrawl`, `searxng`, `linkup` |
| Cost Tracking | ✅ |
| Logging | ✅ |
| Load Balancing | ❌ |
@ -205,7 +205,7 @@ See the [official Perplexity Search documentation](https://docs.perplexity.ai/ap
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `query` | string or array | Yes | Search query. Can be a single string or array of strings |
| `search_provider` | string | Yes (SDK) | The search provider to use: `"perplexity"`, `"tavily"`, `"parallel_ai"`, `"exa_ai"`, `"google_pse"`, `"dataforseo"`, `"firecrawl"`, or `"searxng"` |
| `search_provider` | string | Yes (SDK) | The search provider to use: `"perplexity"`, `"tavily"`, `"parallel_ai"`, `"exa_ai"`, `"google_pse"`, `"dataforseo"`, `"firecrawl"`, `"searxng"`, or `"linkup"` |
| `search_tool_name` | string | Yes (Proxy) | Name of the search tool configured in `config.yaml` |
| `max_results` | integer | No | Maximum number of results to return (1-20). Default: 10 |
| `search_domain_filter` | array | No | List of domains to filter results (max 20 domains) |
@ -269,6 +269,7 @@ The response follows Perplexity's search format with the following structure:
| DataForSEO | `DATAFORSEO_LOGIN`, `DATAFORSEO_PASSWORD` | `dataforseo` |
| Firecrawl | `FIRECRAWL_API_KEY` | `firecrawl` |
| SearXNG | `SEARXNG_API_BASE` (required) | `searxng` |
| Linkup | `LINKUP_API_KEY` | `linkup` |
See the individual provider documentation for detailed setup instructions and provider-specific parameters.

152
docs/my-website/docs/search/linkup.md Normal file
View file

@ -0,0 +1,152 @@
# Linkup Search
**Get API Key:** [https://linkup.so](https://linkup.so)
## LiteLLM Python SDK
```python showLineNumbers title="Linkup Search"
import os
from litellm import search
os.environ["LINKUP_API_KEY"] = "..."
response = search(
query="latest AI developments",
search_provider="linkup",
max_results=5
)
```
## LiteLLM AI Gateway
### 1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
search_tools:
- search_tool_name: linkup-search
litellm_params:
search_provider: linkup
api_key: os.environ/LINKUP_API_KEY
```
### 2. Start the proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Test the search endpoint
```bash showLineNumbers title="Test Request"
curl http://0.0.0.0:4000/v1/search/linkup-search \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"query": "latest AI developments",
"max_results": 5
}'
```
## Provider-specific Parameters
```python showLineNumbers title="Linkup Search with Provider-specific Parameters"
import os
from litellm import search
os.environ["LINKUP_API_KEY"] = "..."
response = search(
query="machine learning research",
search_provider="linkup",
max_results=10,
# Linkup-specific parameters
depth="deep", # "standard" (faster) or "deep" (more comprehensive)
outputType="searchResults", # "searchResults", "sourcedAnswer", or "structured"
includeSources=True, # Include sources in response
includeImages=True, # Include images in results
fromDate="2024-01-01", # Start date filter (YYYY-MM-DD)
toDate="2024-12-31", # End date filter (YYYY-MM-DD)
includeDomains=["arxiv.org", "nature.com"], # Domains to search (max 100)
excludeDomains=["wikipedia.com"], # Domains to exclude
includeInlineCitations=True, # Include inline citations in sourcedAnswer
)
```
## Features
Linkup provides powerful web search with context retrieval capabilities:
### Search Depth
Control the precision and speed of your search:
- `standard` - Returns results faster
- `deep` - Takes longer but yields more comprehensive results
### Output Types
Choose how results are formatted:
- `searchResults` - Returns a list of search results with URLs and content
- `sourcedAnswer` - Returns an AI-generated answer with sources
- `structured` - Returns results in a custom JSON schema format
### Date Filtering
Filter results by date range:
```python
response = search(
query="AI developments",
search_provider="linkup",
fromDate="2024-06-01",
toDate="2024-12-31"
)
```
### Domain Filtering
Include or exclude specific domains:
```python
response = search(
query="research papers",
search_provider="linkup",
includeDomains=["arxiv.org", "nature.com", "ieee.org"],
excludeDomains=["wikipedia.com"]
)
```
### Structured Output
Get results in a custom JSON schema format:
```python
response = search(
query="Microsoft 2024 revenue",
search_provider="linkup",
outputType="structured",
structuredOutputSchema='{"type": "object", "properties": {"revenue": {"type": "string"}, "year": {"type": "string"}}}'
)
```
## Response Format
Linkup returns results in the following format:
```json
{
"results": [
{
"type": "text",
"name": "Microsoft 2024 Annual Report",
"url": "https://www.microsoft.com/investor/reports/ar24/index.html",
"content": "Highlights from fiscal year 2024..."
}
]
}
```
LiteLLM transforms this to the standard `SearchResponse` format:
- `results[].name` → `SearchResult.title`
- `results[].url` → `SearchResult.url`
- `results[].content` → `SearchResult.snippet`

2
docs/my-website/docs/secret_managers/custom_secret_manager.md
View file

@ -76,7 +76,7 @@ docker run -d \
--name litellm-proxy \
-v $(pwd)/config.yaml:/app/config.yaml \
-v $(pwd)/my_secret_manager.py:/app/my_secret_manager.py \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml \
--port 4000 \
--detailed_debug

31
docs/my-website/docs/secret_managers/hashicorp_vault.md
View file

@ -47,6 +47,8 @@ HCP_VAULT_TOKEN="hvs.CAESIG52gL6ljBSdmq*****"
# OPTIONAL
HCP_VAULT_REFRESH_INTERVAL="86400" # defaults to 86400, frequency of cache refresh for Hashicorp Vault
HCP_VAULT_MOUNT_NAME="secret" # OPTIONAL. defaults to "secret", set this if your KV engine is mounted elsewhere
HCP_VAULT_PATH_PREFIX="litellm" # OPTIONAL. defaults to None, set this if your secrets live under a custom prefix like secret/data/litellm/OPENAI_API_KEY
```
**Step 2.** Add to proxy config.yaml
@ -151,18 +153,20 @@ export HCP_VAULT_TOKEN="hvs.CAESIG52gL6ljBSdmq*****"
LiteLLM reads secrets from Hashicorp Vault's KV v2 engine using the following URL format:
```
{VAULT_ADDR}/v1/{NAMESPACE}/secret/data/{SECRET_NAME}
{VAULT_ADDR}/v1/{NAMESPACE}/{MOUNT_NAME}/data/{PATH_PREFIX}/{SECRET_NAME}
```
For example, if you have:
- `HCP_VAULT_ADDR="https://vault.example.com:8200"`
- `HCP_VAULT_NAMESPACE="admin"`
- `HCP_VAULT_MOUNT_NAME="secret"`
- `HCP_VAULT_PATH_PREFIX="litellm"`
- Secret name: `AZURE_API_KEY`
LiteLLM will look up:
```
https://vault.example.com:8200/v1/admin/secret/data/AZURE_API_KEY
https://vault.example.com:8200/v1/admin/secret/data/litellm/AZURE_API_KEY
```
### Expected Secret Format
@ -194,3 +198,26 @@ LiteLLM stores secret under the `prefix_for_stored_virtual_keys` path (default:
<Image img={require('../../img/hcorp_virtual_key.png')} />
### Team-specific overrides
When running the LiteLLM proxy you can override the Vault location per team. Use the [Team-Level Secret Manager Settings](./overview.md#team-level-secret-manager-settings) flow in the dashboard and configure the panel shown below:
<Image img={require('../../img/secret_manager_hashicorp_vault_settings.png')} />
Use the following structure for the JSON payload:
```json
{
"namespace": "teams/team-a",
"mount": "kv-prod",
"path_prefix": "virtual-keys",
"data": "password"
}
```
- `namespace` – overrides the `X-Vault-Namespace` header.
- `mount` – which KV engine mount to use (defaults to `secret`).
- `path_prefix` – additional path segments between the mount and the secret name.
- `data` – the field name inside the KV payload (defaults to `key`).
Whenever LiteLLM stores or deletes virtual keys for that team, these overrides are applied so you can keep each team’s credentials in its own namespace, mount, or field layout without changing the global Vault configuration.

29
docs/my-website/docs/secret_managers/overview.md
View file

@ -1,3 +1,5 @@
import Image from '@theme/IdealImage';
# Secret Managers Overview
:::info
@ -45,3 +47,30 @@ general_settings:
primary_secret_name: "litellm_secrets" # OPTIONAL. Read multiple keys from one JSON secret on AWS Secret Manager
```
## Team-Level Secret Manager Settings
Team-level secret manager settings let every team bring their own key-management configuration. These settings are used when creating virtual keys tied to the team.
Follow these steps to configure it:
1. **Create a team**
Open the Teams page and click `Create Team` to launch the modal.
<Image img={require('../../img/secret_manager_settings_create_team.png')} />
2. **Expand Additional Settings**
Use the `Additional Settings` toggle to reveal the advanced configuration panel.
<Image img={require('../../img/secret_manager_settings_additional_settings.png')} />
3. **Configure the Secret Manager**
In the `Secret Manager Settings` panel, paste the provider-specific JSON. Refer to each provider page (AWS, Azure, Google, Hashicorp, etc.) for the supported keys/values. JSON is required today, but we plan to add a more UI-friendly editor.
<Image img={require('../../img/secret_manager_settings.png')} />
4. **Create the team**
Review the inputs and click `Create Team` to save.
<Image img={require('../../img/secret_manager_settings_create_button.png')} />
Once saved, LiteLLM will use this configuration.

2
docs/my-website/docs/tutorials/elasticsearch_logging.md
View file

@ -221,7 +221,7 @@ services:
- elasticsearch
litellm:
image: ghcr.io/berriai/litellm:main-latest
image: docker.litellm.ai/berriai/litellm:main-latest
ports:
- "4000:4000"
environment:

2
docs/my-website/docs/tutorials/openai_codex.md
View file

@ -53,7 +53,7 @@ yarn global add @openai/codex
docker run \
-v $(pwd)/litellm_config.yaml:/app/config.yaml \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml
```

2
docs/my-website/docusaurus.config.js
View file

@ -8,7 +8,7 @@ const darkCodeTheme = require('prism-react-renderer/themes/dracula');
const inkeepConfig = {
baseSettings: {
apiKey: "0cb9c9916ec71bfe0e53c9d7f83ff046daee3fa9ef318f6a",
apiKey: "test-inkeep-api-key-123",
organizationDisplayName: 'liteLLM',
primaryBrandColor: '#4965f5',
theme: {

BIN
docs/my-website/img/a2a_gateway2.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 16 MiB

BIN
docs/my-website/img/secret_manager_hashicorp_vault_settings.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 107 KiB

BIN
docs/my-website/img/secret_manager_settings.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 680 KiB

BIN
docs/my-website/img/secret_manager_settings_additional_settings.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 683 KiB

BIN
docs/my-website/img/secret_manager_settings_create_button.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 691 KiB

BIN
docs/my-website/img/secret_manager_settings_create_team.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 524 KiB

BIN
docs/my-website/img/sentinel.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 918 KiB

2
docs/my-website/release_notes/v1.55.8-stable/index.md
View file

@ -53,7 +53,7 @@ Send LLM usage (spend, tokens) data to [Azure Data Lake](https://learn.microsoft
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.55.8-stable
docker.litellm.ai/berriai/litellm:litellm_stable_release_branch-v1.55.8-stable
```
## Get Daily Updates

2
docs/my-website/release_notes/v1.57.3/index.md
View file

@ -39,7 +39,7 @@ Instead of `apt-get` use `apk`, the base litellm image will no longer have `apt-
**You are only impacted if you use `apt-get` in your Dockerfile**
```shell
# Use the provided base image
FROM ghcr.io/berriai/litellm:main-latest
FROM docker.litellm.ai/berriai/litellm:main-latest
# Set the working directory
WORKDIR /app

2
docs/my-website/release_notes/v1.63.11-stable/index.md
View file

@ -36,7 +36,7 @@ This release is primarily focused on:
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.63.11-stable
docker.litellm.ai/berriai/litellm:main-v1.63.11-stable
```
## Demo Instance

2
docs/my-website/release_notes/v1.63.14/index.md
View file

@ -32,7 +32,7 @@ This release brings:
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.63.14-stable.patch1
docker.litellm.ai/berriai/litellm:main-v1.63.14-stable.patch1
```
## Demo Instance

2
docs/my-website/release_notes/v1.65.4-stable/index.md
View file

@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.65.4-stable
docker.litellm.ai/berriai/litellm:main-v1.65.4-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.66.0-stable/index.md
View file

@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.66.0-stable
docker.litellm.ai/berriai/litellm:main-v1.66.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.67.4-stable/index.md
View file

@ -30,7 +30,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.67.4-stable
docker.litellm.ai/berriai/litellm:main-v1.67.4-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.68.0-stable/index.md
View file

@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.68.0-stable
docker.litellm.ai/berriai/litellm:main-v1.68.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.69.0-stable/index.md
View file

@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.69.0-stable
docker.litellm.ai/berriai/litellm:main-v1.69.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.70.1-stable/index.md
View file

@ -30,7 +30,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.70.1-stable
docker.litellm.ai/berriai/litellm:main-v1.70.1-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.71.1-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.71.1-stable
docker.litellm.ai/berriai/litellm:main-v1.71.1-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.72.0-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.72.0-stable
docker.litellm.ai/berriai/litellm:main-v1.72.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.72.2-stable/index.md
View file

@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.72.2-stable
docker.litellm.ai/berriai/litellm:main-v1.72.2-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.72.6-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.72.6-stable
docker.litellm.ai/berriai/litellm:main-v1.72.6-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.73.0-stable/index.md
View file

@ -37,7 +37,7 @@ The `non-root` docker image has a known issue around the UI not loading. If you
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.73.0-stable
docker.litellm.ai/berriai/litellm:v1.73.0-stable
```
</TabItem>

Some files were not shown because too many files have changed in this diff Show more