scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge branch 'main' into fix/openrouter-stream-usage-no-stream-options

Browse source
This commit is contained in:
Cesar Garcia 2026-02-27 17:41:13 -03:00 • committed by GitHub
commit acf2fd9828
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1419 changed files with 110040 additions and 21906 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
.circleci/config.yml
View file

@ -1456,6 +1456,7 @@ jobs:
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
pip install "boto3==1.36.0"
pip install "semantic_router==0.1.10"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@ -4099,6 +4100,63 @@ jobs:
path: playwright-report
destination: playwright-report
prisma_schema_sync:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Install Neon CLI
command: |
npm i -g neonctl
- run:
name: Install curl and dockerize
command: |
sudo apt-get update
sudo apt-get install -y curl
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Sync schema on base e2e database
command: |
BASE_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch br-fancy-paper-ad1olsb3 \
--database-name yuneng-trial-db \
--role neondb_owner)
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$BASE_DATABASE_URL \
-e LITELLM_MASTER_KEY="sk-1234" \
--name schema-sync \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--use_prisma_db_push
- run:
name: Start outputting logs
command: docker logs -f schema-sync
background: true
- run:
name: Wait for proxy to be ready (schema sync complete)
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Stop schema sync container
command: docker stop schema-sync
test_nonroot_image:
machine:
image: ubuntu-2204:2023.10.1
@ -4297,6 +4355,15 @@ workflows:
only:
- main
- /litellm_.*/
- prisma_schema_sync:
context: e2e_ui_tests
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- e2e_ui_testing:
name: e2e_ui_testing_chromium
browser: chromium
@ -4304,6 +4371,7 @@ workflows:
requires:
- ui_build
- build_docker_database_image
- prisma_schema_sync
filters:
branches:
only:
@ -4316,6 +4384,7 @@ workflows:
requires:
- ui_build
- build_docker_database_image
- prisma_schema_sync
filters:
branches:
only:

2
.github/ISSUE_TEMPLATE/config.yml vendored
View file

@ -1,7 +1,7 @@
blank_issues_enabled: true
contact_links:
- name: Schedule Demo
url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat
url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions
about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM
- name: Discord
url: https://discord.com/invite/wuPM9dRgDw

4
.github/workflows/check_duplicate_issues.yml vendored
View file

@ -20,10 +20,10 @@ jobs:
reaction: eyes
comment: |
**⚠️ Potential duplicate detected**
This issue appears similar to existing issue(s):
{{#issues}}
- [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
{{/issues}}
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.

2
.github/workflows/interpret_load_test.py vendored
View file

@ -123,7 +123,7 @@ if __name__ == "__main__":
+ docker_run_command
+ "\n\n"
+ "### Don't want to maintain your internal proxy? get in touch 🎉"
+ "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
+ "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions"
+ "\n\n"
+ "## Load Test LiteLLM Proxy Results"
+ "\n\n"

80
.github/workflows/regenerate-poetry-lock.yml vendored Normal file
View file

@ -0,0 +1,80 @@
name: Regenerate poetry.lock
# Runs whenever pyproject.toml is merged into main (the most common cause of
# the "pyproject.toml changed significantly since poetry.lock was last generated"
# CI failure). Can also be triggered manually.
on:
push:
branches:
- main
paths:
- pyproject.toml
workflow_dispatch:
permissions:
contents: write # needed to push the auto/regenerate-poetry-lock-* branch
pull-requests: write # needed to open the PR and enable auto-merge
jobs:
regenerate-lock:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install Poetry
run: pip install poetry
- name: Regenerate poetry.lock
run: poetry lock
- name: Check whether poetry.lock actually changed
id: diff
run: |
if git diff --quiet poetry.lock; then
echo "changed=false" >> "$GITHUB_OUTPUT"
else
echo "changed=true" >> "$GITHUB_OUTPUT"
fi
- name: Open PR with the refreshed lock file
if: steps.diff.outputs.changed == 'true'
id: open-pr
run: |
BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')"
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git checkout -b "$BRANCH"
git add poetry.lock
git commit -m "chore: regenerate poetry.lock to match pyproject.toml"
git push -f origin "$BRANCH"
cat > /tmp/pr-body.md << 'BODY'
Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`.
Fixes the recurring CI failure:
```
pyproject.toml changed significantly since poetry.lock was last generated.
Run `poetry lock` to fix the lock file.
```
BODY
PR_URL=$(gh pr create \
--title "chore: regenerate poetry.lock to match pyproject.toml" \
--body-file /tmp/pr-body.md \
--head "$BRANCH" \
--base main)
echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
env:
GH_TOKEN: ${{ github.token }}
- name: Enable auto-merge
if: steps.diff.outputs.changed == 'true'
run: |
gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash
env:
GH_TOKEN: ${{ github.token }}

60
.github/workflows/test-litellm-matrix.yml vendored
View file

@ -48,8 +48,19 @@ jobs:
path: "tests/test_litellm/litellm_core_utils"
workers: 2
reruns: 1
- name: "other"
path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types"
- name: "other-1"
# responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
workers: 2
reruns: 2
- name: "other-2"
# enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
workers: 2
reruns: 2
- name: "other-3"
# remaining dirs ≈ 8.0k lines
path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
workers: 2
reruns: 2
- name: "root"
@ -57,12 +68,49 @@ jobs:
workers: 2
reruns: 2
# tests/proxy_unit_tests split alphabetically (~48 files total)
- name: "proxy-unit-a"
path: "tests/proxy_unit_tests/test_[a-o]*.py"
- name: "proxy-unit-a1"
# test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
path: "tests/proxy_unit_tests/test_[a-j]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b"
path: "tests/proxy_unit_tests/test_[p-z]*.py"
- name: "proxy-unit-a2"
# test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
path: "tests/proxy_unit_tests/test_[k-o]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b1"
# lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b2"
# proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
path: "tests/proxy_unit_tests/test_proxy_server.py"
workers: 2
reruns: 1
- name: "proxy-unit-b3"
# proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
workers: 2
reruns: 1
- name: "proxy-unit-b4"
# proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
path: "tests/proxy_unit_tests/test_proxy_utils.py"
workers: 2
reruns: 1
- name: "proxy-unit-b5"
# proxy_token_counter (1279) - runs independently from utils
path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
workers: 2
reruns: 1
- name: "proxy-unit-b6"
# test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
path: "tests/proxy_unit_tests/test_[r-t]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b7"
# test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
path: "tests/proxy_unit_tests/test_[u-z]*.py"
workers: 2
reruns: 1

39
AGENTS.md
View file

@ -174,6 +174,8 @@ When opening issues or pull requests, follow these templates:
3. **Rate Limits**: Respect provider rate limits in tests
4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
5. **Dependencies**: Keep dependencies minimal and well-justified
6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
## HELPFUL RESOURCES
@ -187,4 +189,39 @@ When opening issues or pull requests, follow these templates:
- Check similar provider implementations
- Ensure comprehensive test coverage
- Update documentation appropriately
- Consider backward compatibility impact
- Consider backward compatibility impact
## Cursor Cloud specific instructions
### Environment
- Poetry is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
- Python 3.12, Node 22 are pre-installed.
- The virtual environment lives under `~/.cache/pypoetry/virtualenvs/`.
### Running the proxy server
Start the proxy with a config file:
```bash
poetry run litellm --config dev_config.yaml --port 4000
```
The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
### Running tests
See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
- The `--timeout` pytest flag is NOT available; don't pass it.
- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
- Black `--check` may report pre-existing formatting issues; this does not block test runs.
### Lint
```bash
cd litellm && poetry run ruff check .
```
Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.

4
CLAUDE.md
View file

@ -97,6 +97,10 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- Integration tests for each provider in `tests/llm_translation/`
- Proxy tests in `tests/proxy_unit_tests/`
- Load tests in `tests/load_tests/`
- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
### UI / Backend Consistency
- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
### Database Migrations
- Prisma handles schema migrations

20
Dockerfile
View file

@ -49,7 +49,7 @@ USER root
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
# SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
# levels inside its dependency tree. `npm install -g <pkg>` only creates a
# SEPARATE global package, it does NOT replace npm's internal copies.
@ -64,6 +64,12 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
npm cache clean --force
WORKDIR /app
@ -90,14 +96,20 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done
# Install semantic_router and aurelio-sdk using script

4
README.md
View file

@ -203,7 +203,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
{
"mcpServers": {
"LiteLLM": {
"url": "http://localhost:4000/mcp",
"url": "http://localhost:4000/mcp/",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
@ -399,7 +399,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
# Enterprise
For companies that need better security, user management and professional support
[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
This covers:
- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**

3
ci_cd/security_scans.sh
View file

@ -158,6 +158,9 @@ run_grype_scans() {
"CVE-2025-11468" # No fix available yet
"CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
"CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
"GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
"GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
)
# Build JSON array of allowlisted CVE IDs for jq

2
cookbook/benchmark/readme.md
View file

@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?':
```
## Support
**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.

119
cookbook/gollem_go_agent_framework/README.md Normal file
View file

@ -0,0 +1,119 @@
# Gollem Go Agent Framework with LiteLLM
A working example showing how to use [gollem](https://github.com/fugue-labs/gollem), a production-grade Go agent framework, with LiteLLM as a proxy gateway. This lets Go developers access 100+ LLM providers through a single proxy while keeping compile-time type safety for tools and structured output.
## Quick Start
### 1. Start LiteLLM Proxy
```bash
# Simple start with a single model
litellm --model gpt-4o
# Or with the example config for multi-provider access
litellm --config proxy_config.yaml
```
### 2. Run the examples
```bash
# Install Go dependencies
go mod tidy
# Basic agent
go run ./basic
# Agent with type-safe tools
go run ./tools
# Streaming responses
go run ./streaming
```
## Configuration
The included `proxy_config.yaml` sets up three providers through LiteLLM:
```yaml
model_list:
- model_name: gpt-4o # OpenAI
- model_name: claude-sonnet # Anthropic
- model_name: gemini-pro # Google Vertex AI
```
Switch providers in Go by changing a single string — no code changes needed:
```go
model := openai.NewLiteLLM("http://localhost:4000",
openai.WithModel("gpt-4o"), // OpenAI
// openai.WithModel("claude-sonnet"), // Anthropic
// openai.WithModel("gemini-pro"), // Google
)
```
## Examples
### `basic/` — Basic Agent
Connects gollem to LiteLLM and runs a simple prompt. Demonstrates the `NewLiteLLM` constructor and basic agent creation.
### `tools/` — Type-Safe Tools
Shows gollem's compile-time type-safe tool framework working through LiteLLM's tool-use passthrough. The tool parameters are Go structs with JSON tags — the schema is generated automatically at compile time.
### `streaming/` — Streaming Responses
Real-time token streaming using Go 1.23+ range-over-function iterators, proxied through LiteLLM's SSE passthrough.
## How It Works
Gollem's `openai.NewLiteLLM()` constructor creates an OpenAI-compatible provider pointed at your LiteLLM proxy. Since LiteLLM speaks the OpenAI API protocol, everything works out of the box:
- **Chat completions** — standard request/response
- **Tool use** — LiteLLM passes tool definitions and calls through transparently
- **Streaming** — Server-Sent Events proxied through LiteLLM
- **Structured output** — JSON schema response format works with supporting models
```
Go App (gollem) → LiteLLM Proxy → OpenAI / Anthropic / Google / ...
```
## Why Use This?
- **Type-safe Go**: Compile-time type checking for tools, structured output, and agent configuration — no runtime surprises
- **Single proxy, many models**: Switch between OpenAI, Anthropic, Google, and 100+ other providers by changing a model name string
- **Zero-dependency core**: gollem's core has no external dependencies — just stdlib
- **Single binary deployment**: `go build` produces one binary, no pip/venv/Docker needed
- **Cost tracking & rate limiting**: LiteLLM handles cost tracking, rate limits, and fallbacks at the proxy layer
## Environment Variables
```bash
# Required for providers you want to use (set in LiteLLM config or env)
export OPENAI_API_KEY="sk-..."
export ANTHROPIC_API_KEY="sk-ant-..."
# Optional: point to a non-default LiteLLM proxy
export LITELLM_PROXY_URL="http://localhost:4000"
```
## Troubleshooting
**Connection errors?**
- Make sure LiteLLM is running: `litellm --model gpt-4o`
- Check the URL is correct (default: `http://localhost:4000`)
**Model not found?**
- Verify the model name matches what's configured in LiteLLM
- Run `curl http://localhost:4000/models` to see available models
**Tool calls not working?**
- Ensure the underlying model supports tool use (GPT-4o, Claude, Gemini)
- Check LiteLLM logs for any provider-specific errors
## Learn More
- [gollem GitHub](https://github.com/fugue-labs/gollem)
- [gollem API Reference](https://pkg.go.dev/github.com/fugue-labs/gollem/core)
- [LiteLLM Proxy Docs](https://docs.litellm.ai/docs/simple_proxy)
- [LiteLLM Supported Models](https://docs.litellm.ai/docs/providers)

41
cookbook/gollem_go_agent_framework/basic/main.go Normal file
View file

@ -0,0 +1,41 @@
// Basic gollem agent connected to a LiteLLM proxy.
//
// Usage:
//
// litellm --model gpt-4o # start proxy in another terminal
// go run ./basic
package main
import (
"context"
"fmt"
"log"
"os"
"github.com/fugue-labs/gollem/core"
"github.com/fugue-labs/gollem/provider/openai"
)
func main() {
proxyURL := "http://localhost:4000"
if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
proxyURL = u
}
// Connect to LiteLLM proxy. NewLiteLLM creates an OpenAI-compatible
// provider pointed at the given URL.
model := openai.NewLiteLLM(proxyURL,
openai.WithModel("gpt-4o"), // any model name configured in LiteLLM
)
// Create and run a simple agent.
agent := core.NewAgent[string](model,
core.WithSystemPrompt[string]("You are a helpful assistant. Be concise."),
)
result, err := agent.Run(context.Background(), "Explain quantum computing in two sentences.")
if err != nil {
log.Fatal(err)
}
fmt.Println(result.Output)
}

5
cookbook/gollem_go_agent_framework/go.mod Normal file
View file

@ -0,0 +1,5 @@
module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework
go 1.25.1
require github.com/fugue-labs/gollem v0.1.0

2
cookbook/gollem_go_agent_framework/go.sum Normal file
View file

@ -0,0 +1,2 @@
github.com/fugue-labs/gollem v0.1.0 h1:QexYnvkb44QZFEljgAePqMIGZjgsbk0Y5GJ2jYYgfa8=
github.com/fugue-labs/gollem v0.1.0/go.mod h1:htW1YO81uysSKVOkYJtxhGCFrzm+36HBFxEWuECoHKQ=

16
cookbook/gollem_go_agent_framework/proxy_config.yaml Normal file
View file

@ -0,0 +1,16 @@
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: gemini-pro
litellm_params:
model: vertex_ai/gemini-2.0-flash
vertex_project: my-project
vertex_location: us-central1

56
cookbook/gollem_go_agent_framework/streaming/main.go Normal file
View file

@ -0,0 +1,56 @@
// Streaming responses from gollem through LiteLLM.
//
// Uses Go 1.23+ range-over-function iterators for real-time token
// streaming via LiteLLM's SSE passthrough.
//
// Usage:
//
// litellm --model gpt-4o
// go run ./streaming
package main
import (
"context"
"fmt"
"log"
"os"
"github.com/fugue-labs/gollem/core"
"github.com/fugue-labs/gollem/provider/openai"
)
func main() {
proxyURL := "http://localhost:4000"
if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
proxyURL = u
}
model := openai.NewLiteLLM(proxyURL,
openai.WithModel("gpt-4o"),
)
agent := core.NewAgent[string](model)
// RunStream returns a streaming result that yields tokens as they arrive.
stream, err := agent.RunStream(context.Background(), "Write a haiku about distributed systems")
if err != nil {
log.Fatal(err)
}
// StreamText yields text chunks in real-time.
// The boolean argument controls whether deltas (true) or accumulated
// text (false) is returned.
fmt.Print("Response: ")
for text, err := range stream.StreamText(true) {
if err != nil {
log.Fatal(err)
}
fmt.Print(text)
}
fmt.Println()
// After streaming completes, the final response is available.
resp := stream.Response()
fmt.Printf("\nTokens used: input=%d, output=%d\n",
resp.Usage.InputTokens, resp.Usage.OutputTokens)
}

64
cookbook/gollem_go_agent_framework/tools/main.go Normal file
View file

@ -0,0 +1,64 @@
// Gollem agent with type-safe tools through LiteLLM.
//
// The tool parameters are Go structs — gollem generates the JSON schema
// automatically at compile time. LiteLLM passes tool definitions through
// transparently to the underlying provider.
//
// Usage:
//
// litellm --model gpt-4o
// go run ./tools
package main
import (
"context"
"fmt"
"log"
"os"
"github.com/fugue-labs/gollem/core"
"github.com/fugue-labs/gollem/provider/openai"
)
// WeatherParams defines the tool's input schema via struct tags.
// The JSON schema is generated at compile time — no runtime reflection needed.
type WeatherParams struct {
City string `json:"city" description:"City name to get weather for"`
Unit string `json:"unit,omitempty" description:"Temperature unit: celsius or fahrenheit"`
}
func main() {
proxyURL := "http://localhost:4000"
if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
proxyURL = u
}
model := openai.NewLiteLLM(proxyURL,
openai.WithModel("gpt-4o"),
)
// Define a type-safe tool. The function signature enforces correct types.
weatherTool := core.FuncTool[WeatherParams](
"get_weather",
"Get current weather for a city",
func(ctx context.Context, p WeatherParams) (string, error) {
unit := p.Unit
if unit == "" {
unit = "fahrenheit"
}
// In production, call a real weather API here.
return fmt.Sprintf("Weather in %s: 72°F (22°C), sunny", p.City), nil
},
)
agent := core.NewAgent[string](model,
core.WithTools[string](weatherTool),
core.WithSystemPrompt[string]("You are a helpful weather assistant. Use the get_weather tool to answer weather questions."),
)
result, err := agent.Run(context.Background(), "What's the weather like in San Francisco and Tokyo?")
if err != nil {
log.Fatal(err)
}
fmt.Println(result.Output)
}

4
deploy/charts/litellm-helm/README.md
View file

@ -36,6 +36,10 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` |
| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` |
| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` |
| `livenessProbe.*` | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` |
| `readinessProbe.*` | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` |
| `startupProbe.*` | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` |
| `resources.*` | CPU/memory requests and limits for the LiteLLM container. | `{}` |
| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` |
| `ingress.labels` | Additional labels for the Ingress resource | `{}` |
| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A |

2
deploy/charts/litellm-helm/templates/configmap-litellm.yaml
View file

@ -6,4 +6,4 @@ metadata:
data:
config.yaml: |
{{ .Values.proxy_config | toYaml | indent 6 }}
{{- end }}
{{- end }}

25
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -158,18 +158,31 @@ spec:
{{- end }}
livenessProbe:
httpGet:
path: /health/liveliness
path: {{ .Values.livenessProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
successThreshold: {{ .Values.livenessProbe.successThreshold }}
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
readinessProbe:
httpGet:
path: /health/readiness
path: {{ .Values.readinessProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
successThreshold: {{ .Values.readinessProbe.successThreshold }}
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
startupProbe:
httpGet:
path: /health/readiness
path: {{ .Values.startupProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
failureThreshold: 30
periodSeconds: 10
initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.startupProbe.periodSeconds }}
timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
successThreshold: {{ .Values.startupProbe.successThreshold }}
failureThreshold: {{ .Values.startupProbe.failureThreshold }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
@ -235,4 +248,4 @@ spec:
{{- if .Values.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
{{- end }}
{{- end }}

148
deploy/charts/litellm-helm/tests/deployment_tests.yaml
View file

@ -159,4 +159,150 @@ tests:
value: -c
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
value: echo "Container stopping"
value: echo "Container stopping"
- it: should render background health check settings from proxy_config.general_settings
template: configmap-litellm.yaml
set:
proxy_config.general_settings.background_health_checks: true
proxy_config.general_settings.health_check_interval: 240
proxy_config.general_settings.health_check_concurrency: 16
proxy_config.general_settings.health_check_details: false
asserts:
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*background_health_checks:\s*true$'
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*health_check_interval:\s*240$'
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*health_check_concurrency:\s*16$'
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*health_check_details:\s*false$'
- it: should allow overriding liveness, readiness, and startup probes
template: deployment.yaml
set:
livenessProbe:
path: /custom/livez
initialDelaySeconds: 5
periodSeconds: 15
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
path: /custom/readyz
initialDelaySeconds: 10
periodSeconds: 20
timeoutSeconds: 6
successThreshold: 1
failureThreshold: 6
startupProbe:
path: /custom/startupz
initialDelaySeconds: 15
periodSeconds: 25
timeoutSeconds: 7
successThreshold: 1
failureThreshold: 40
asserts:
- equal:
path: spec.template.spec.containers[0].livenessProbe.httpGet.path
value: /custom/livez
- equal:
path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
value: 5
- equal:
path: spec.template.spec.containers[0].readinessProbe.httpGet.path
value: /custom/readyz
- equal:
path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
value: 6
- equal:
path: spec.template.spec.containers[0].startupProbe.httpGet.path
value: /custom/startupz
- equal:
path: spec.template.spec.containers[0].startupProbe.failureThreshold
value: 40
- it: should render container resources from values
template: deployment.yaml
set:
resources:
limits:
cpu: 500m
memory: 2Gi
requests:
cpu: 250m
memory: 1Gi
asserts:
- equal:
path: spec.template.spec.containers[0].resources.limits.cpu
value: 500m
- equal:
path: spec.template.spec.containers[0].resources.limits.memory
value: 2Gi
- equal:
path: spec.template.spec.containers[0].resources.requests.cpu
value: 250m
- equal:
path: spec.template.spec.containers[0].resources.requests.memory
value: 1Gi
- it: should keep default probes and empty resources unchanged
template: deployment.yaml
asserts:
- equal:
path: spec.template.spec.containers[0].livenessProbe.httpGet.path
value: /health/liveliness
- equal:
path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds
value: 0
- equal:
path: spec.template.spec.containers[0].livenessProbe.periodSeconds
value: 10
- equal:
path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
value: 1
- equal:
path: spec.template.spec.containers[0].livenessProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].livenessProbe.failureThreshold
value: 3
- equal:
path: spec.template.spec.containers[0].readinessProbe.httpGet.path
value: /health/readiness
- equal:
path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds
value: 0
- equal:
path: spec.template.spec.containers[0].readinessProbe.periodSeconds
value: 10
- equal:
path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
value: 1
- equal:
path: spec.template.spec.containers[0].readinessProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].readinessProbe.failureThreshold
value: 3
- equal:
path: spec.template.spec.containers[0].startupProbe.httpGet.path
value: /health/readiness
- equal:
path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds
value: 0
- equal:
path: spec.template.spec.containers[0].startupProbe.periodSeconds
value: 10
- equal:
path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
value: 1
- equal:
path: spec.template.spec.containers[0].startupProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].startupProbe.failureThreshold
value: 30
- equal:
path: spec.template.spec.containers[0].resources
value: {}

25
deploy/charts/litellm-helm/values.yaml
View file

@ -84,6 +84,31 @@ service:
separateHealthApp: false
separateHealthPort: 8081
# Probe tuning for proxy container
livenessProbe:
path: /health/liveliness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
readinessProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
startupProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 30
ingress:
enabled: false
className: "nginx"

23
docker/Dockerfile.custom_ui
View file

@ -5,8 +5,21 @@ FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
WORKDIR /app
# Install Node.js and npm (adjust version as needed)
RUN apt-get update && apt-get install -y nodejs npm && \
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
RUN apt-get update && apt-get upgrade -y \
libxml2 \
libexpat1 \
openssl \
libssl3 \
git \
libkrb5-3 \
libglib2.0-0 \
wget \
libaom3 \
libxslt1.1 \
libgnutls30 \
libc6 && \
apt-get install -y nodejs npm && \
npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -17,6 +30,12 @@ RUN apt-get update && apt-get install -y nodejs npm && \
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
npm cache clean --force
# Copy the UI source into the container

20
docker/Dockerfile.database
View file

@ -50,7 +50,7 @@ USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -61,6 +61,12 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
npm cache clean --force
WORKDIR /app
@ -79,14 +85,20 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done
# Install semantic_router and aurelio-sdk using script

37
docker/Dockerfile.dev
View file

@ -56,13 +56,26 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# Install only runtime dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
libssl3 \
RUN apt-get update && apt-get upgrade -y \
libxml2 \
libexpat1 \
openssl \
libssl3 \
git \
libkrb5-3 \
libglib2.0-0 \
wget \
libaom3 \
libxslt1.1 \
libgnutls30 \
libc6 \
&& apt-get install -y --no-install-recommends \
libssl3 \
libatomic1 \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
&& npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -73,6 +86,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done \
&& npm cache clean --force
WORKDIR /app
@ -95,14 +114,20 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done
# Generate prisma client and set permissions

23
docker/Dockerfile.non_root
View file

@ -80,7 +80,7 @@ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
XDG_CACHE_HOME=/app/.cache \
PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.12.0 \
RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.13.1 \
&& mkdir -p /app/.cache/npm
RUN NPM_CONFIG_CACHE=/app/.cache/npm \
@ -105,7 +105,8 @@ RUN for i in 1 2 3; do \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done \
&& npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
&& apk upgrade --no-cache nodejs \
&& npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -116,6 +117,12 @@ RUN for i in 1 2 3; do \
&& find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done \
&& npm cache clean --force
# Copy artifacts from builder
@ -162,14 +169,20 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done
# Permissions, cleanup, and Prisma prep

147
docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md Normal file
View file

@ -0,0 +1,147 @@
---
slug: anthropic-wildcard-model-access-incident
title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload"
date: 2026-02-23T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
tags: [incident-report, proxy, auth, model-access]
hide_table_of_contents: false
---
**Date:** Feb 23, 2026
**Duration:** ~3 hours
**Severity:** High (for users with provider wildcard access rules)
**Status:** Resolved
## Summary
When a new Anthropic model (e.g. `claude-sonnet-4-6`) was added to the LiteLLM model cost map and a cost map reload was triggered, requests to the new model were rejected with:
```
key not allowed to access model. This key can only access models=['anthropic/*']. Tried to access claude-sonnet-4-6.
```
The reload updated `litellm.model_cost` correctly but never re-ran `add_known_models()`, so `litellm.anthropic_models` (the in-memory set used by the wildcard resolver) remained stale. The new model was invisible to the `anthropic/*` wildcard even though the cost map knew about it.
- **LLM calls:** All requests to newly-added Anthropic models were blocked with a 401.
- **Existing models:** Unaffected — only models missing from the stale provider set were impacted.
- **Other providers:** Same bug class existed for any provider wildcard (e.g. `openai/*`, `gemini/*`).
{/* truncate */}
---
## Background
LiteLLM supports provider-level wildcard access rules. When an admin configures a key or team with `models=['anthropic/*']`, any model whose provider resolves to `anthropic` should be allowed. The resolution happens in `_model_custom_llm_provider_matches_wildcard_pattern`:
```mermaid
flowchart TD
A["1. Request arrives for claude-sonnet-4-6"] --> B["2. Auth check: can this key call this model?
proxy/auth/auth_checks.py"]
B --> C["3. Key has models=['anthropic/*']
→ wildcard match attempted"]
C --> D["4. get_llm_provider('claude-sonnet-4-6')
checks litellm.anthropic_models set"]
D -->|"model IN set"| E["5a. ✅ Provider = 'anthropic'
→ 'anthropic/claude-sonnet-4-6' matches 'anthropic/*'"]
D -->|"model NOT IN set"| F["5b. ❌ Provider unknown
→ exception raised → wildcard returns False"]
E --> G["6. Request allowed"]
F --> H["6. 401: key not allowed to access model"]
style E fill:#d4edda,stroke:#28a745
style F fill:#f8d7da,stroke:#dc3545
style H fill:#f8d7da,stroke:#dc3545
style D fill:#fff3cd,stroke:#ffc107
```
`litellm.anthropic_models` is a Python `set` populated at import time by `add_known_models()`. It is the source `get_llm_provider()` consults to map a bare model name like `claude-sonnet-4-6` to the provider string `"anthropic"`.
---
## Root Cause
`add_known_models()` is called **once** at module import time. Both reload paths in `proxy_server.py` updated `litellm.model_cost` with the fresh map but never called `add_known_models()` again:
```python
# Before the fix — both reload paths looked like this:
new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
litellm.model_cost = new_model_cost_map # ✅ cost map updated
_invalidate_model_cost_lowercase_map() # ✅ cache cleared
# ❌ add_known_models() never called
# → litellm.anthropic_models still has the old set
# → new model not in the set
# → get_llm_provider() raises for the new model
# → wildcard match returns False
# → 401 for every request to the new model
```
The gap existed in two places:
1. `_check_and_reload_model_cost_map` — the periodic automatic reload (every 10 s)
2. The `/reload/model_cost_map` admin endpoint — the manual reload
**Timeline:**
1. New model (`claude-sonnet-4-6`) added to `model_prices_and_context_window.json`
2. Admin triggers cost map reload via UI → `litellm.model_cost` updated
3. Users with `anthropic/*` wildcard keys attempt requests to `claude-sonnet-4-6`
4. `get_llm_provider('claude-sonnet-4-6')` raises → wildcard returns False → 401
5. Admin reloads cost map again — same result (root cause not addressed)
6. ~3 hours of investigation → root cause identified → fix deployed
---
## The Fix
After each reload, `add_known_models()` is called with the freshly fetched map passed explicitly. Passing the map directly (rather than relying on the module-level reference) removes any ambiguity about which dict is iterated:
```python
# After the fix — both reload paths now do:
new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
litellm.model_cost = new_model_cost_map
_invalidate_model_cost_lowercase_map()
litellm.add_known_models(model_cost_map=new_model_cost_map) # ✅ sets repopulated
```
`add_known_models()` was also updated to accept an optional explicit map so callers cannot accidentally iterate a stale module-level reference:
```python
# Before
def add_known_models():
for key, value in model_cost.items(): # reads module global — ambiguous after reload
...
# After
def add_known_models(model_cost_map: Optional[Dict] = None):
_map = model_cost_map if model_cost_map is not None else model_cost
for key, value in _map.items(): # always iterates the map you just fetched
...
```
After the fix, the provider sets (`anthropic_models`, `open_ai_chat_completion_models`, etc.) are always consistent with `litellm.model_cost` immediately after every reload. New models become accessible via wildcard rules without any proxy restart.
---
## Remediation
| # | Action | Status | Code |
|---|---|---|---|
| 1 | Call `add_known_models(model_cost_map=...)` in the periodic reload path | ✅ Done | [`proxy_server.py#L4393`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L4393) |
| 2 | Call `add_known_models(model_cost_map=...)` in the `/reload/model_cost_map` endpoint | ✅ Done | [`proxy_server.py#L11904`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L11904) |
| 3 | Update `add_known_models()` to accept an explicit map parameter | ✅ Done | [`__init__.py#L617`](https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py#L617) |
| 4 | Regression test: `add_known_models(model_cost_map=...)` populates provider sets | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
| 5 | Regression test: `anthropic/*` wildcard grants/denies access correctly after reload | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
---

4
docs/my-website/blog/gemin_3.1/index.md
View file

@ -37,7 +37,7 @@ LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.80.8-stable.1
ghcr.io/berriai/litellm:main-v1.81.9-stable.gemini.3.1-pro
```
</TabItem>
@ -45,7 +45,7 @@ ghcr.io/berriai/litellm:main-v1.80.8-stable.1
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.80.8.post1
pip install litellm==v1.81.9-stable.gemini.3.1-pro
```
</TabItem>

145
docs/my-website/blog/gpt_5_3_codex/index.md Normal file
View file

@ -0,0 +1,145 @@
---
slug: gpt_5_3_codex
title: "Day 0 Support: GPT-5.3-Codex"
date: 2026-02-24T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API."
tags: [openai, gpt-5.3-codex, codex, day 0 support]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items.
## Why `phase` matters for GPT-5.3-Codex
`phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses.
Reference: [Phase parameter docs](https://developers.openai.com/api/reference/overview)
Supported values:
- `null`
- `"commentary"`
- `"final_answer"`
Important:
- Persist assistant output items with `phase` exactly as returned.
- Send those assistant items back on the next turn.
- Do **not** add `phase` to user messages.
## Docker Image
```bash
docker pull ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3
```
## Usage
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gpt-5.3-codex
litellm_params:
model: openai/gpt-5.3-codex
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e ANTHROPIC_API_KEY=$OPENAI_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3 \
--config /app/config.yaml
```
**3. Test it**
```bash
curl -X POST "http://0.0.0.0:4000/v1/responses" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gpt-5.3-codex",
"input": "Write a Python script that checks if a number is prime."
}'
```
</TabItem>
</Tabs>
## Python Example: Persist `phase` with OpenAI Client + LiteLLM Base URL
```python
from openai import OpenAI
client = OpenAI(
base_url="http://0.0.0.0:4000/v1", # LiteLLM Proxy
api_key="your-litellm-api-key",
)
items = [] # Persist this per conversation/thread
def _item_get(item, key, default=None):
if isinstance(item, dict):
return item.get(key, default)
return getattr(item, key, default)
def run_turn(user_text: str):
global items
# User message: no phase field
items.append(
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": user_text}],
}
)
resp = client.responses.create(
model="gpt-5.3-codex",
input=items,
)
# Persist assistant output items verbatim, including phase
for out_item in (resp.output or []):
items.append(out_item)
# Optional: inspect latest phase for UI/telemetry routing
latest_phase = None
for out_item in reversed(resp.output or []):
if _item_get(out_item, "type") == "output_item.done" and _item_get(out_item, "phase") is not None:
latest_phase = _item_get(out_item, "phase")
break
return resp, latest_phase
```
## Notes
- Use `/v1/responses` for GPT Codex models.
- Preserve full assistant output history for best multi-turn behavior.
- If `phase` metadata is dropped during history reconstruction, output quality can degrade on long-running tasks.

154
docs/my-website/blog/server_root_path/index.md Normal file
View file

@ -0,0 +1,154 @@
---
slug: server-root-path-incident
title: "Incident Report: SERVER_ROOT_PATH regression broke UI routing"
date: 2026-02-21T10:00:00
authors:
- name: Yuneng Jiang
title: SWE @ LiteLLM (Full Stack)
url: https://www.linkedin.com/in/yunengjiang/
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
tags: [incident-report, ui, stability]
hide_table_of_contents: false
---
**Date:** January 22, 2026
**Duration:** ~4 days (until fix merged January 26, 2026)
**Severity:** High
**Status:** Resolved
> **Note:** This fix is available starting from LiteLLM `v1.81.3.rc.6` or higher.
## Summary
A PR ([`#19467`](https://github.com/BerriAI/litellm/pull/19467)) accidentally removed the `root_path=server_root_path` parameter from the FastAPI app initialization in `proxy_server.py`. This caused the proxy to ignore the `SERVER_ROOT_PATH` environment variable when serving the UI. Users who deploy LiteLLM behind a reverse proxy with a path prefix (e.g., `/api/v1` or `/llmproxy`) found that all UI pages returned 404 Not Found.
- **LLM API calls:** No impact. API routing was unaffected.
- **UI pages:** All UI pages returned 404 for deployments using `SERVER_ROOT_PATH`.
- **Swagger/OpenAPI docs:** Broken when accessed through the configured root path.
{/* truncate */}
---
## Background
Many LiteLLM deployments run behind a reverse proxy (e.g., Nginx, Traefik, AWS ALB) that routes traffic to LiteLLM under a path prefix. FastAPI's `root_path` parameter tells the application about this prefix so it can correctly serve static files, generate URLs, and handle routing.
```mermaid
sequenceDiagram
participant User as User Browser
participant RP as Reverse Proxy
participant LP as LiteLLM Proxy
User->>RP: GET /llmproxy/ui/
RP->>LP: GET /ui/ (X-Forwarded-Prefix: /llmproxy)
Note over LP: Before regression:<br/>FastAPI root_path="/llmproxy"<br/>→ Serves UI correctly
Note over LP: After regression:<br/>FastAPI root_path=""<br/>→ UI assets resolve to wrong paths<br/>→ 404 Not Found
```
The `root_path` parameter was present in `proxy_server.py` since early versions of LiteLLM. It was removed as a side effect of PR [#19467](https://github.com/BerriAI/litellm/pull/19467), which was intended to fix a different UI 404 issue.
---
## Root cause
PR [#19467](https://github.com/BerriAI/litellm/pull/19467) (`73d49f8`) removed the `root_path=server_root_path` line from the `FastAPI()` constructor in `proxy_server.py`:
```diff
app = FastAPI(
docs_url=_get_docs_url(),
redoc_url=_get_redoc_url(),
title=_title,
description=_description,
version=version,
- root_path=server_root_path,
lifespan=proxy_startup_event,
)
```
Without `root_path`, FastAPI treated all requests as if the application was mounted at `/`, causing path mismatches for any deployment using `SERVER_ROOT_PATH`.
The regression went undetected because:
1. **No automated test** verified that `root_path` was set on the FastAPI app.
2. **No manual test procedure** existed for `SERVER_ROOT_PATH` functionality.
3. **Default deployments** (without `SERVER_ROOT_PATH`) were unaffected, so most CI tests passed.
---
## Remediation
| # | Action | Status | Code |
| --- | ------------------------------------------------------------------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------- |
| 1 | Restore `root_path=server_root_path` in FastAPI app initialization | ✅ Done | [`#19790`](https://github.com/BerriAI/litellm/pull/19790) (`5426b3c`) |
| 2 | Add unit tests for `get_server_root_path()` and FastAPI app initialization | ✅ Done | [`test_server_root_path.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_server_root_path.py) |
| 3 | Add CI workflow that builds Docker image and tests UI routing with `SERVER_ROOT_PATH` on every PR | ✅ Done | [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) |
| 4 | Document manual test procedure for `SERVER_ROOT_PATH` | ✅ Done | [Discussion #8495](https://github.com/BerriAI/litellm/discussions/8495) |
---
## CI workflow details
The new [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) workflow runs on every PR against `main`. It:
1. Builds the LiteLLM Docker image
2. Starts a container with `SERVER_ROOT_PATH` set (tests both `/api/v1` and `/llmproxy`)
3. Verifies the UI returns valid HTML at `{ROOT_PATH}/ui/`
4. Fails the workflow if the UI is unreachable
```mermaid
flowchart TD
A["PR opened/updated"] --> B["Build Docker image"]
B --> C["Start container with SERVER_ROOT_PATH=/api/v1"]
B --> D["Start container with SERVER_ROOT_PATH=/llmproxy"]
C --> E["curl {ROOT_PATH}/ui/ → expect HTML"]
D --> F["curl {ROOT_PATH}/ui/ → expect HTML"]
E -->|"HTML found"| G["✅ Pass"]
E -->|"404 or no HTML"| H["❌ Fail Workflow"]
F -->|"HTML found"| G
F -->|"404 or no HTML"| H
style G fill:#d4edda,stroke:#28a745
style H fill:#f8d7da,stroke:#dc3545
```
This prevents future regressions where changes to `proxy_server.py` accidentally break `SERVER_ROOT_PATH` support.
---
## Timeline
| Time (UTC) | Event |
| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| Jan 22, 2026 04:20 | PR [#19467](https://github.com/BerriAI/litellm/pull/19467) merged, removing `root_path=server_root_path` |
| Jan 22–26 | Users on nightly builds report UI 404 errors when using `SERVER_ROOT_PATH` |
| Jan 26, 2026 17:48 | Fix PR [#19790](https://github.com/BerriAI/litellm/pull/19790) merged, restoring `root_path=server_root_path` |
| Feb 18, 2026 | CI workflow [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) added to run on every PR |
---
## Resolution steps for users
For users still experiencing issues, update to the latest LiteLLM version:
```bash
pip install --upgrade litellm
```
Verify your `SERVER_ROOT_PATH` is correctly set:
```bash
# In your environment or docker-compose.yml
SERVER_ROOT_PATH="/your-prefix"
```
Then confirm the UI is accessible at `http://your-host:4000/your-prefix/ui/`.

38
docs/my-website/docs/benchmarks.md
View file

@ -5,6 +5,44 @@ import Image from '@theme/IdealImage';
Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpoint.
## Setting Up Benchmarking with Network Mock
The fastest way to benchmark proxy overhead is using `network_mock` mode. This intercepts outbound requests at the httpx transport layer and returns canned responses, no need for setting up a mock provider.
**1. Create a proxy config:**
```yaml
model_list:
- model_name: db-openai-endpoint
litellm_params:
model: openai/gpt-4o
api_key: "sk-fake-key"
api_base: "https://api.openai.com"
litellm_settings:
network_mock: true
callbacks: []
num_retries: 0
request_timeout: 30
general_settings:
master_key: "sk-1234"
```
**2. Start the proxy:**
```bash
litellm --config benchmark_config.yaml --port 4000 --num_workers 8
```
**3. Run the benchmark script:**
```bash
python scripts/benchmark_mock.py --requests 2000 --max-concurrent 200 --runs 3
```
This measures pure proxy overhead on the hot path without any network latency to a real or fake provider.
## Setting Up a Fake OpenAI Endpoint
For load testing and benchmarking, you can use a fake OpenAI proxy server. LiteLLM provides:

2
docs/my-website/docs/caching/all_caches.md
View file

@ -297,6 +297,7 @@ litellm.cache = Cache(
similarity_threshold=0.7, # similarity threshold for cache hits, 0 == no similarity, 1 = exact matches, 0.5 == 50% similarity
qdrant_quantization_config ="binary", # can be one of 'binary', 'product' or 'scalar' quantizations that is supported by qdrant
qdrant_semantic_cache_embedding_model="text-embedding-ada-002", # this model is passed to litellm.embedding(), any litellm.embedding() model is supported here
qdrant_semantic_cache_vector_size=1536, # vector size for the embedding model, must match the dimensionality of the embedding model used
)
response1 = completion(
@ -635,6 +636,7 @@ def __init__(
qdrant_quantization_config: Optional[str] = None,
qdrant_semantic_cache_embedding_model="text-embedding-ada-002",
qdrant_semantic_cache_vector_size: Optional[int] = None,
**kwargs
):
```

82
docs/my-website/docs/completion/prompt_caching.md
View file

@ -63,7 +63,6 @@ for _ in range(2):
}
],
},
# marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
{
"role": "user",
"content": [
@ -77,7 +76,6 @@ for _ in range(2):
"role": "assistant",
"content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
},
# The final turn is marked with cache-control, for continuing in followups.
{
"role": "user",
"content": [
@ -112,16 +110,16 @@ model_list:
api_key: os.environ/OPENAI_API_KEY
```
2. Start proxy
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
3. Test it!
```python
from openai import OpenAI
from openai import OpenAI
import os
client = OpenAI(
@ -144,7 +142,6 @@ for _ in range(2):
}
],
},
# marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
{
"role": "user",
"content": [
@ -158,7 +155,6 @@ for _ in range(2):
"role": "assistant",
"content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
},
# The final turn is marked with cache-control, for continuing in followups.
{
"role": "user",
"content": [
@ -183,6 +179,78 @@ assert response.usage.prompt_tokens_details.cached_tokens > 0
</TabItem>
</Tabs>
### OpenAI `prompt_cache_key` and `prompt_cache_retention`
OpenAI prompt caching is [**automatic**](https://platform.openai.com/docs/guides/prompt-caching) — no `cache_control` message annotations are needed. Any request with 1024+ prompt tokens is eligible for caching.
OpenAI also supports two optional parameters for more control over caching behavior:
- **`prompt_cache_key`** (string) — A routing hint that improves cache hit rates for requests sharing long common prefixes. Requests with the same cache key are routed to the same backend, increasing the likelihood of a cache hit.
- **`prompt_cache_retention`** (`"in_memory"` or `"24h"`) — Controls cache TTL. Default is `"in_memory"` (5–10 min). Set to `"24h"` for extended caching that offloads KV tensors to GPU-local storage.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ["OPENAI_API_KEY"] = ""
response = completion(
model="gpt-4o",
messages=[
{
"role": "system",
"content": "You are an AI assistant tasked with analyzing legal documents. "
+ "Here is the full text of a complex legal agreement " * 400,
},
{
"role": "user",
"content": "What are the key terms and conditions?",
},
],
prompt_cache_key="legal-doc-analysis",
prompt_cache_retention="24h",
)
print(response.usage)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```python
from openai import OpenAI
client = OpenAI(
api_key="LITELLM_PROXY_KEY",
base_url="LITELLM_PROXY_BASE",
)
response = client.chat.completions.create(
model="gpt-4o",
messages=[
{
"role": "system",
"content": "You are an AI assistant tasked with analyzing legal documents. "
+ "Here is the full text of a complex legal agreement " * 400,
},
{
"role": "user",
"content": "What are the key terms and conditions?",
},
],
extra_body={
"prompt_cache_key": "legal-doc-analysis",
"prompt_cache_retention": "24h",
},
)
print(response.usage)
```
</TabItem>
</Tabs>
### Anthropic Example
Anthropic charges for cache writes.

22
docs/my-website/docs/contributing.md
View file

@ -79,7 +79,27 @@ cp -r out/* ../../litellm/proxy/_experimental/out/
Then restart the proxy and access the UI at `http://localhost:4000/ui`
## 4. Submitting a PR
## 4. Pre-PR Checklist
Before submitting your pull request, make sure the following pass locally from `ui/litellm-dashboard/`:
**Run tests related to your changes:**
```bash
npx vitest run src/components/path/to/YourComponent.test.tsx
```
Tests are co-located with components (e.g., `TeamInfo.tsx` → `TeamInfo.test.tsx`). If you add a new component, add a corresponding `.test.tsx` file next to it.
**Run the build:**
```bash
npm run build
```
These map to the `ui_tests` and `ui_build` CI checks.
## 5. Submitting a PR
1. Create a new branch for your changes:
```bash

6
docs/my-website/docs/enterprise.md
View file

@ -4,7 +4,7 @@ import Image from '@theme/IdealImage';
:::info
- ✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) to discuss your needs.
- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) to discuss your needs.
:::
For companies that need SSO, user management and professional support for LiteLLM Proxy
@ -36,7 +36,7 @@ Manage Yourself - you can deploy our Docker Image or build a custom image from o
### What’s the cost of the Self-Managed Enterprise edition?
Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
### How does deployment with Enterprise License work?
@ -106,7 +106,7 @@ Professional Support can assist with LLM/Provider integrations, deployment, upgr
Pricing is based on usage. We can figure out a price that works for your team, on the call.
[**Contact Us to learn more**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[**Contact Us to learn more**](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)

2
docs/my-website/docs/fine_tuning.md
View file

@ -6,7 +6,7 @@ import TabItem from '@theme/TabItem';
:::info
This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

1
docs/my-website/docs/generateContent.md
View file

@ -15,6 +15,7 @@ Use LiteLLM to call Google AI's generateContent endpoints for text generation, m
| Streaming | ✅ | |
| Fallbacks | ✅ | between supported models |
| Loadbalancing | ✅ | between supported models |
| Metadata Tracking | ✅ | passes trace ID, metadata to observability callbacks (e.g. S3, Langfuse) |
## Usage
---

18
docs/my-website/docs/interactions.md
View file

@ -130,13 +130,12 @@ Point the Google GenAI SDK to LiteLLM Proxy:
```python showLineNumbers title="Google GenAI SDK with LiteLLM Proxy"
from google import genai
import os
# Point SDK to LiteLLM Proxy
os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
os.environ["GEMINI_API_KEY"] = "sk-1234" # Your LiteLLM API key
client = genai.Client()
client = genai.Client(
api_key="sk-1234", # Your LiteLLM API key
http_options={"base_url": "http://localhost:4000"},
)
# Create an interaction
interaction = client.interactions.create(
@ -151,12 +150,11 @@ print(interaction.outputs[-1].text)
```python showLineNumbers title="Google GenAI SDK Streaming"
from google import genai
import os
os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
os.environ["GEMINI_API_KEY"] = "sk-1234"
client = genai.Client()
client = genai.Client(
api_key="sk-1234", # Your LiteLLM API key
http_options={"base_url": "http://localhost:4000"},
)
for chunk in client.interactions.create_stream(
model="gemini/gemini-2.5-flash",

2
docs/my-website/docs/mcp.md
View file

@ -641,7 +641,7 @@ import asyncio
config = {
"mcpServers": {
"mcp_group": {
"url": "http://localhost:4000/mcp",
"url": "http://localhost:4000/mcp/",
"headers": {
"x-mcp-servers": "dev_group", # assume this gives access to github, zapier and deepwiki
"x-litellm-api-key": "Bearer sk-1234",

2
docs/my-website/docs/observability/gcs_bucket_integration.md
View file

@ -6,7 +6,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage?
:::info
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

98
docs/my-website/docs/ocr.md
View file

@ -61,6 +61,52 @@ async def test_async_ocr():
asyncio.run(test_async_ocr())
```
### Using Local Files
LiteLLM can read local files directly — no manual base64 encoding needed:
```python
from litellm import ocr
# OCR with a local PDF file path
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "file",
"file": "/path/to/document.pdf"
}
)
# OCR with a file object
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "file",
"file": open("document.pdf", "rb")
}
)
# OCR with raw bytes
with open("document.pdf", "rb") as f:
pdf_bytes = f.read()
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "file",
"file": pdf_bytes,
"mime_type": "application/pdf" # recommended for raw bytes (auto-detected from extension for file paths)
}
)
```
The `file` field accepts:
- **File path** (`str` or `pathlib.Path`) — LiteLLM reads the file and detects the MIME type from the extension
- **File object** (binary file-like object) — e.g. `open("doc.pdf", "rb")`
- **Raw bytes** (`bytes`) — use `mime_type` to specify the content type
LiteLLM automatically converts file inputs to base64 data URIs internally, so all providers work seamlessly.
### Using Base64 Encoded Documents
```python
@ -121,7 +167,7 @@ litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
Test request
**Test request — JSON body**
```bash
curl http://0.0.0.0:4000/v1/ocr \
@ -136,6 +182,27 @@ curl http://0.0.0.0:4000/v1/ocr \
}'
```
**Test request — multipart file upload**
Upload a file directly using multipart form data. No need to base64-encode the file yourself.
```bash
curl http://0.0.0.0:4000/v1/ocr \
-H "Authorization: Bearer sk-1234" \
-F "model=mistral-ocr" \
-F "file=@/path/to/document.pdf"
```
You can also pass optional parameters as additional form fields:
```bash
curl http://0.0.0.0:4000/v1/ocr \
-H "Authorization: Bearer sk-1234" \
-F "model=mistral-ocr" \
-F "file=@screenshot.png" \
-F 'pages=[0,1,2]' \
-F "include_image_base64=true"
```
## **Request/Response Format**
@ -168,10 +235,12 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) |
| `document` | object | Yes | Document to process. Must contain `type` and URL field |
| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images |
| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) |
| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) |
| `document` | object | Yes | Document to process. Must contain `type` and the corresponding field |
| `document.type` | string | Yes | `"document_url"` for PDFs/docs, `"image_url"` for images, or `"file"` for local files |
| `document.document_url` | string | Conditional | URL or data URI to the document (required if `type` is `"document_url"`) |
| `document.image_url` | string | Conditional | URL or data URI to the image (required if `type` is `"image_url"`) |
| `document.file` | string/bytes/file | Conditional | File path, bytes, or file-like object (required if `type` is `"file"`) |
| `document.mime_type` | string | No | Explicit MIME type for file inputs (auto-detected from extension if not provided) |
| `pages` | array | No | List of specific page indices to process (0-indexed) |
| `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings |
| `image_limit` | integer | No | Maximum number of images to return |
@ -179,7 +248,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
#### Document Format Examples
**For PDFs and documents:**
**For PDFs and documents (URL):**
```json
{
"type": "document_url",
@ -187,7 +256,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
}
```
**For images:**
**For images (URL):**
```json
{
"type": "image_url",
@ -203,6 +272,21 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
}
```
**For local files (SDK):**
```python
{"type": "file", "file": "/path/to/document.pdf"}
{"type": "file", "file": open("image.png", "rb")}
{"type": "file", "file": pdf_bytes, "mime_type": "application/pdf"}
```
**For file uploads (Proxy — multipart form):**
```bash
curl http://0.0.0.0:4000/v1/ocr \
-H "Authorization: Bearer sk-1234" \
-F "model=mistral-ocr" \
-F "file=@document.pdf"
```
### Response Format
The response follows Mistral's OCR format with the following structure:

71
docs/my-website/docs/pass_through/google_ai_studio.md
View file

@ -35,26 +35,25 @@ curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:countTokens?key=
```
</TabItem>
<TabItem value="js" label="Google AI Node.js SDK">
<TabItem value="js" label="Google GenAI JS SDK">
```javascript
const { GoogleGenerativeAI } = require("@google/generative-ai");
const { GoogleGenAI } = require("@google/genai");
const modelParams = {
model: 'gemini-pro',
};
const requestOptions = {
baseUrl: 'http://localhost:4000/gemini', // http://<proxy-base-url>/gemini
};
const genAI = new GoogleGenerativeAI("sk-1234"); // litellm proxy API key
const model = genAI.getGenerativeModel(modelParams, requestOptions);
const ai = new GoogleGenAI({
apiKey: "sk-1234", // litellm proxy API key
httpOptions: {
baseUrl: "http://localhost:4000/gemini", // http://<proxy-base-url>/gemini
},
});
async function main() {
try {
const result = await model.generateContent("Explain how AI works");
console.log(result.response.text());
const response = await ai.models.generateContent({
model: "gemini-2.5-flash",
contents: "Explain how AI works",
});
console.log(response.text);
} catch (error) {
console.error('Error:', error);
}
@ -63,12 +62,13 @@ async function main() {
// For streaming responses
async function main_streaming() {
try {
const streamingResult = await model.generateContentStream("Explain how AI works");
for await (const chunk of streamingResult.stream) {
console.log('Stream chunk:', JSON.stringify(chunk));
const response = await ai.models.generateContentStream({
model: "gemini-2.5-flash",
contents: "Explain how AI works",
});
for await (const chunk of response) {
process.stdout.write(chunk.text);
}
const aggregatedResponse = await streamingResult.response;
console.log('Aggregated response:', JSON.stringify(aggregatedResponse));
} catch (error) {
console.error('Error:', error);
}
@ -321,29 +321,28 @@ curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:generateContent?
```
</TabItem>
<TabItem value="js" label="Google AI Node.js SDK">
<TabItem value="js" label="Google GenAI JS SDK">
```javascript
const { GoogleGenerativeAI } = require("@google/generative-ai");
const { GoogleGenAI } = require("@google/genai");
const modelParams = {
model: 'gemini-pro',
};
const requestOptions = {
baseUrl: 'http://localhost:4000/gemini', // http://<proxy-base-url>/gemini
customHeaders: {
"tags": "gemini-js-sdk,pass-through-endpoint"
}
};
const genAI = new GoogleGenerativeAI("sk-1234");
const model = genAI.getGenerativeModel(modelParams, requestOptions);
const ai = new GoogleGenAI({
apiKey: "sk-1234",
httpOptions: {
baseUrl: "http://localhost:4000/gemini", // http://<proxy-base-url>/gemini
headers: {
"tags": "gemini-js-sdk,pass-through-endpoint",
},
},
});
async function main() {
try {
const result = await model.generateContent("Explain how AI works");
console.log(result.response.text());
const response = await ai.models.generateContent({
model: "gemini-2.5-flash",
contents: "Explain how AI works",
});
console.log(response.text);
} catch (error) {
console.error('Error:', error);
}

1
docs/my-website/docs/providers/groq.md
View file

@ -159,6 +159,7 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion
| moonshotai/kimi-k2-instruct-0905 | `completion(model="groq/moonshotai/kimi-k2-instruct-0905", messages)` |
| openai/gpt-oss-120b | `completion(model="groq/openai/gpt-oss-120b", messages)` |
| openai/gpt-oss-20b | `completion(model="groq/openai/gpt-oss-20b", messages)` |
| openai/gpt-oss-safeguard-20b | `completion(model="groq/openai/gpt-oss-safeguard-20b", messages)` |
## Groq - Tool / Function Calling Example

121
docs/my-website/docs/providers/perplexity.md
View file

@ -120,7 +120,7 @@ All models listed here https://docs.perplexity.ai/docs/model-cards are supported
## Agentic Research API (Responses API)
## Agent API (Responses API)
Requires v1.72.6+
@ -196,7 +196,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
model="perplexity/openai/gpt-5.2",
input="Explain quantum computing in simple terms",
custom_llm_provider="perplexity",
max_output_tokens=500,
@ -215,7 +215,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/anthropic/claude-3-5-sonnet-20241022",
model="perplexity/anthropic/claude-sonnet-4-5",
input="Write a short story about a robot learning to paint",
custom_llm_provider="perplexity",
max_output_tokens=500,
@ -234,7 +234,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/google/gemini-2.0-flash-exp",
model="perplexity/google/gemini-2.5-flash",
input="Explain the concept of neural networks",
custom_llm_provider="perplexity",
max_output_tokens=500,
@ -253,7 +253,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/xai/grok-2-1212",
model="perplexity/xai/grok-4-1-fast-non-reasoning",
input="What makes a good AI assistant?",
custom_llm_provider="perplexity",
max_output_tokens=500,
@ -276,7 +276,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
model="perplexity/openai/gpt-5.2",
input="What's the weather in San Francisco today?",
custom_llm_provider="perplexity",
tools=[{"type": "web_search"}],
@ -286,6 +286,78 @@ response = responses(
print(response.output)
```
### Function Calling
The Agent API supports custom function tools. Pass function tools through unchanged:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-5.2",
input="What's the weather in San Francisco?",
custom_llm_provider="perplexity",
tools=[
{"type": "web_search"},
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a location",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
},
},
},
},
],
instructions="Use tools when appropriate.",
)
print(response.output)
```
### Structured Outputs
Request JSON schema structured outputs via the `text` parameter:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/preset/pro-search",
input="Extract key facts about the Eiffel Tower",
custom_llm_provider="perplexity",
text={
"format": {
"type": "json_schema",
"name": "facts",
"schema": {
"type": "object",
"properties": {
"name": {"type": "string"},
"height_meters": {"type": "number"},
"year_built": {"type": "integer"},
},
"required": ["name", "height_meters", "year_built"],
},
"strict": True,
}
},
)
print(response.output)
```
### Reasoning Effort (Responses API)
@ -319,7 +391,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/anthropic/claude-3-5-sonnet-20241022",
model="perplexity/anthropic/claude-sonnet-4-5",
input=[
{"type": "message", "role": "system", "content": "You are a helpful assistant."},
{"type": "message", "role": "user", "content": "What are the latest AI developments?"},
@ -343,7 +415,7 @@ import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
model="perplexity/openai/gpt-5.2",
input="Tell me a story about space exploration",
custom_llm_provider="perplexity",
stream=True,
@ -360,23 +432,28 @@ for chunk in response:
| Provider | Model Name | Function Call |
|----------|------------|---------------|
| OpenAI | gpt-4o | `responses(model="perplexity/openai/gpt-4o", ...)` |
| OpenAI | gpt-4o-mini | `responses(model="perplexity/openai/gpt-4o-mini", ...)` |
| OpenAI | gpt-5.2 | `responses(model="perplexity/openai/gpt-5.2", ...)` |
| Anthropic | claude-3-5-sonnet-20241022 | `responses(model="perplexity/anthropic/claude-3-5-sonnet-20241022", ...)` |
| Anthropic | claude-3-5-haiku-20241022 | `responses(model="perplexity/anthropic/claude-3-5-haiku-20241022", ...)` |
| Google | gemini-2.0-flash-exp | `responses(model="perplexity/google/gemini-2.0-flash-exp", ...)` |
| Google | gemini-2.0-flash-thinking-exp | `responses(model="perplexity/google/gemini-2.0-flash-thinking-exp", ...)` |
| xAI | grok-2-1212 | `responses(model="perplexity/xai/grok-2-1212", ...)` |
| xAI | grok-2-vision-1212 | `responses(model="perplexity/xai/grok-2-vision-1212", ...)` |
| OpenAI | gpt-5.1 | `responses(model="perplexity/openai/gpt-5.1", ...)` |
| OpenAI | gpt-5-mini | `responses(model="perplexity/openai/gpt-5-mini", ...)` |
| Anthropic | claude-opus-4-6 | `responses(model="perplexity/anthropic/claude-opus-4-6", ...)` |
| Anthropic | claude-opus-4-5 | `responses(model="perplexity/anthropic/claude-opus-4-5", ...)` |
| Anthropic | claude-sonnet-4-5 | `responses(model="perplexity/anthropic/claude-sonnet-4-5", ...)` |
| Anthropic | claude-haiku-4-5 | `responses(model="perplexity/anthropic/claude-haiku-4-5", ...)` |
| Google | gemini-3-pro-preview | `responses(model="perplexity/google/gemini-3-pro-preview", ...)` |
| Google | gemini-3-flash-preview | `responses(model="perplexity/google/gemini-3-flash-preview", ...)` |
| Google | gemini-2.5-pro | `responses(model="perplexity/google/gemini-2.5-pro", ...)` |
| Google | gemini-2.5-flash | `responses(model="perplexity/google/gemini-2.5-flash", ...)` |
| xAI | grok-4-1-fast-non-reasoning | `responses(model="perplexity/xai/grok-4-1-fast-non-reasoning", ...)` |
| Perplexity | sonar | `responses(model="perplexity/perplexity/sonar", ...)` |
### Available Presets
| Preset Name | Function Call |
|----------------|--------------------------------------------------------|
| fast-search | `responses(model="perplexity/preset/fast-search", ...)`|
| pro-search | `responses(model="perplexity/preset/pro-search", ...)` |
| deep-research | `responses(model="perplexity/preset/deep-research", ...)`|
| Preset Name | Function Call |
|-------------|---------------|
| fast-search | `responses(model="perplexity/preset/fast-search", ...)` |
| pro-search | `responses(model="perplexity/preset/pro-search", ...)` |
| deep-research | `responses(model="perplexity/preset/deep-research", ...)` |
| advanced-deep-research | `responses(model="perplexity/preset/advanced-deep-research", ...)` |
### Complete Example
@ -388,7 +465,7 @@ os.environ['PERPLEXITY_API_KEY'] = ""
# Comprehensive example with multiple features
response = responses(
model="perplexity/openai/gpt-4o",
model="perplexity/openai/gpt-5.2",
input="Research the latest developments in quantum computing and provide sources",
custom_llm_provider="perplexity",
tools=[

203
docs/my-website/docs/providers/vertex_realtime.md Normal file
View file

@ -0,0 +1,203 @@
# Vertex AI Gemini Live - Realtime API
Use Vertex AI's Gemini Live API (BidiGenerateContent) through LiteLLM's unified `/realtime` endpoint, which speaks the OpenAI Realtime protocol.
| Feature | Supported |
|---------|-----------|
| Proxy (`/realtime`) | ✅ |
| Voice in / Voice out | ✅ |
| Text in / Text out | ✅ |
| Server VAD | ✅ |
| Output transcription | ✅ |
## Setup
### 1. Auth
LiteLLM uses your Google Cloud credentials (OAuth2 Bearer token), not an API key.
```bash
gcloud auth application-default login
```
Or set a service-account key file:
```bash
export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa-key.json
```
### 2. Proxy config
```yaml
model_list:
- model_name: vertex-gemini-live
litellm_params:
model: vertex_ai/gemini-2.0-flash-live-001
vertex_project: your-gcp-project-id
vertex_location: us-east4 # or any supported region, or "global"
general_settings:
master_key: sk-your-key
```
### 3. Start the proxy
```bash
litellm --config config.yaml --port 4000
```
## Usage
### Python (websockets)
```python
import asyncio
import json
import websockets
PROXY_URL = "ws://localhost:4000/realtime?model=vertex-gemini-live"
API_KEY = "sk-your-key"
async def main():
async with websockets.connect(
PROXY_URL,
additional_headers={"api-key": API_KEY},
) as ws:
# Wait for session.created
event = json.loads(await ws.recv())
print(f"session.created: {event['session']['id']}")
# Send a text message
await ws.send(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "Say hello in one sentence."}],
},
}))
# Collect the response
async for raw in ws:
ev = json.loads(raw)
t = ev.get("type", "")
if t == "response.text.delta":
print(ev.get("delta", ""), end="", flush=True)
elif t == "response.done":
print("\n[done]")
break
asyncio.run(main())
```
### Node.js
```js
const WebSocket = require("ws");
const ws = new WebSocket(
"ws://localhost:4000/realtime?model=vertex-gemini-live",
{ headers: { "api-key": "sk-your-key" } }
);
ws.on("open", () => {
ws.send(JSON.stringify({
type: "conversation.item.create",
item: {
type: "message",
role: "user",
content: [{ type: "input_text", text: "Say hello." }],
},
}));
});
ws.on("message", (data) => {
const ev = JSON.parse(data);
if (ev.type === "response.text.delta") process.stdout.write(ev.delta);
if (ev.type === "response.done") ws.close();
});
```
### OpenAI SDK (Python)
```python
import asyncio
from openai import AsyncOpenAI
client = AsyncOpenAI(
base_url="http://localhost:4000",
api_key="sk-your-key",
)
async def main():
async with client.beta.realtime.connect(
model="vertex-gemini-live"
) as conn:
await conn.session.update(session={"modalities": ["text"]})
await conn.conversation.item.create(
item={
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "Say hello."}],
}
)
async for event in conn:
if event.type == "response.text.delta":
print(event.delta, end="", flush=True)
elif event.type == "response.done":
print()
break
asyncio.run(main())
```
## Voice in / Voice out
For a complete voice example see [`voice_realtime_test.py`](https://github.com/BerriAI/litellm/blob/main/voice_realtime_test.py).
Key settings for audio:
- Microphone input: **16 kHz** PCM16 (`audio/pcm;rate=16000`)
- Speaker output: **24 kHz** PCM16 (Vertex AI returns audio at 24 kHz)
- Server VAD is enabled by default with 800 ms silence threshold
```python
# session.update with server VAD — the proxy ignores this for Vertex AI
# because VAD is already configured in the initial setup message.
await ws.send(json.dumps({
"type": "session.update",
"session": {
"modalities": ["audio"],
"turn_detection": {"type": "server_vad", "silence_duration_ms": 800},
},
}))
```
## Supported OpenAI Realtime Events
**Client → Proxy (→ Vertex AI)**
| OpenAI event | Notes |
|---|---|
| `input_audio_buffer.append` | Forwarded as `realtime_input.audio` |
| `conversation.item.create` | Forwarded as `realtime_input.text` |
| `session.update` | Silently ignored — Vertex AI does not support mid-session reconfiguration |
| `response.create` | Silently ignored — Vertex AI responds automatically after each turn |
**Vertex AI → Proxy (→ Client)**
| OpenAI event emitted | Vertex AI source |
|---|---|
| `session.created` | Synthesized after `setupComplete` |
| `response.text.delta` | `serverContent.modelTurn.parts[].text` |
| `response.audio.delta` | `serverContent.modelTurn.parts[].inlineData` |
| `response.audio_transcript.delta` | `serverContent.outputTranscription.text` |
| `conversation.item.input_audio_transcription.completed` | `serverContent.inputTranscription.text` |
| `response.done` | `serverContent.turnComplete` |
## Limitations
- `session.update` is not forwarded (Vertex AI only accepts one setup message per connection).
- Tool calling / function calling is not yet supported.
- Audio transcription requires `outputAudioTranscription: {}` to be set in the initial setup (done automatically by LiteLLM).

53
docs/my-website/docs/proxy/alerting.md
View file

@ -438,6 +438,59 @@ curl -X GET --location 'http://0.0.0.0:4000/health/services?service=webhook' \
- `event_message` *str*: A human-readable description of the event.
### Digest Mode (Reducing Alert Noise)
By default, LiteLLM sends a separate Slack message for **every** alert event. For high-frequency alert types like `llm_requests_hanging` or `llm_too_slow`, this can produce hundreds of duplicate messages per day.
**Digest mode** aggregates duplicate alerts within a configurable time window and emits a single summary message with the total count and time range.
#### Configuration
Use `alert_type_config` in `general_settings` to enable digest mode per alert type:
```yaml
general_settings:
alerting: ["slack"]
alert_type_config:
llm_requests_hanging:
digest: true
digest_interval: 86400 # 24 hours (default)
llm_too_slow:
digest: true
digest_interval: 3600 # 1 hour
llm_exceptions:
digest: true
# uses default interval (86400 seconds / 24 hours)
```
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `digest` | bool | `false` | Enable digest mode for this alert type |
| `digest_interval` | int | `86400` (24h) | Time window in seconds. Alerts are aggregated within this interval. |
#### How It Works
1. When an alert fires for a digest-enabled type, it is **grouped** by `(alert_type, request_model, api_base)` instead of being sent immediately
2. A counter tracks how many times the alert fires within the interval
3. When the interval expires, a **single summary message** is sent:
```
Alert type: `llm_requests_hanging` (Digest)
Level: `Medium`
Start: `2026-02-19 03:27:39`
End: `2026-02-20 03:27:39`
Count: `847`
Message: `Requests are hanging - 600s+ request time`
Request Model: `gemini-2.5-flash`
API Base: `None`
```
#### Limitations
- **Per-instance**: Digest state is held in memory per proxy instance. If you run multiple instances (e.g., Cloud Run with autoscaling), each instance maintains its own digest and emits its own summary.
- **Not durable**: If an instance is terminated before the digest interval expires, the aggregated alerts for that instance are lost.
## Region-outage alerting (✨ Enterprise feature)
:::info

186
docs/my-website/docs/proxy/auto_routing.md
View file

@ -219,3 +219,189 @@ curl -X POST http://localhost:4000/v1/chat/completions \
3. If a route's similarity score exceeds the threshold, the request is routed to that model
4. If no route matches, the request goes to the default model
---
## Complexity Router
The Complexity Router provides an alternative to semantic routing that uses **rule-based scoring** to classify requests by complexity and route them to appropriate models — with **zero external API calls** and **sub-millisecond latency**.
### When to Use
| Feature | Semantic Auto Router | Complexity Router |
|---------|---------------------|-------------------|
| Classification | Embedding-based matching | Rule-based scoring |
| Latency | ~100-500ms (embedding API) | &lt;1ms |
| API Calls | Requires embedding model | None |
| Training | Requires utterance examples | Works out of the box |
| Best For | Intent-based routing | Cost optimization |
Use **Complexity Router** when you want to:
- Route simple queries to cheaper/faster models (e.g., gpt-4o-mini)
- Route complex queries to more capable models (e.g., claude-sonnet-4)
- Minimize latency overhead from routing decisions
- Avoid additional API costs for embeddings
### LiteLLM Python SDK
```python
from litellm import Router
router = Router(
model_list=[
# Target models for each tier
{
"model_name": "gpt-4o-mini",
"litellm_params": {"model": "gpt-4o-mini"},
},
{
"model_name": "gpt-4o",
"litellm_params": {"model": "gpt-4o"},
},
{
"model_name": "claude-sonnet",
"litellm_params": {"model": "claude-sonnet-4-20250514"},
},
{
"model_name": "o1-preview",
"litellm_params": {"model": "o1-preview"},
},
# Complexity router configuration
{
"model_name": "smart-router",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"tiers": {
"SIMPLE": "gpt-4o-mini",
"MEDIUM": "gpt-4o",
"COMPLEX": "claude-sonnet",
"REASONING": "o1-preview",
},
},
"complexity_router_default_model": "gpt-4o",
},
},
],
)
```
#### Usage
```python
# Simple query → routes to gpt-4o-mini
response = await router.acompletion(
model="smart-router",
messages=[{"role": "user", "content": "What is 2+2?"}],
)
# Complex technical query → routes to claude-sonnet or higher
response = await router.acompletion(
model="smart-router",
messages=[{"role": "user", "content": "Design a distributed microservice architecture with Kubernetes orchestration"}],
)
# Reasoning request → routes to o1-preview
response = await router.acompletion(
model="smart-router",
messages=[{"role": "user", "content": "Think step by step and reason through this problem carefully..."}],
)
```
### LiteLLM Proxy Server
Add the complexity router to your `config.yaml`:
```yaml
model_list:
# Target models
- model_name: gpt-4o-mini
litellm_params:
model: gpt-4o-mini
- model_name: gpt-4o
litellm_params:
model: gpt-4o
- model_name: claude-sonnet
litellm_params:
model: claude-sonnet-4-20250514
- model_name: o1-preview
litellm_params:
model: o1-preview
# Complexity router
- model_name: smart-router
litellm_params:
model: auto_router/complexity_router
complexity_router_config:
tiers:
SIMPLE: gpt-4o-mini
MEDIUM: gpt-4o
COMPLEX: claude-sonnet
REASONING: o1-preview
complexity_router_default_model: gpt-4o
```
### Configuration Options
#### Tier Boundaries
Customize the score thresholds for each tier:
```yaml
complexity_router_config:
tiers:
SIMPLE: gpt-4o-mini
MEDIUM: gpt-4o
COMPLEX: claude-sonnet
REASONING: o1-preview
tier_boundaries:
simple_medium: 0.15 # Below 0.15 → SIMPLE
medium_complex: 0.35 # 0.15-0.35 → MEDIUM
complex_reasoning: 0.60 # 0.35-0.60 → COMPLEX, above → REASONING
```
#### Token Thresholds
Adjust when prompts are considered "short" or "long":
```yaml
complexity_router_config:
token_thresholds:
simple: 15 # Prompts under 15 tokens are penalized (simple indicator)
complex: 400 # Prompts over 400 tokens get complexity boost
```
#### Dimension Weights
Customize how much each signal contributes to the complexity score:
```yaml
complexity_router_config:
dimension_weights:
tokenCount: 0.10 # Prompt length
codePresence: 0.30 # Code-related keywords
reasoningMarkers: 0.25 # "step by step", "think through", etc.
technicalTerms: 0.25 # Domain-specific complexity
simpleIndicators: 0.05 # "what is", "define", greetings
multiStepPatterns: 0.03 # "first...then", numbered steps
questionComplexity: 0.02 # Multiple questions
```
### How Complexity Routing Works
The router scores each request across 7 dimensions:
| Dimension | What It Detects | Effect |
|-----------|-----------------|--------|
| Token Count | Short (&lt;15) or long (&gt;400) prompts | Short = simple, long = complex |
| Code Presence | "function", "class", "api", "database", etc. | Increases complexity |
| Reasoning Markers | "step by step", "think through", "analyze" | Triggers REASONING tier |
| Technical Terms | "architecture", "distributed", "encryption" | Increases complexity |
| Simple Indicators | "what is", "define", "hello" | Decreases complexity |
| Multi-Step Patterns | "first...then", "1. 2. 3." | Increases complexity |
| Question Complexity | Multiple question marks | Increases complexity |
**Special behavior:** If 2+ reasoning markers are detected in the user message, the request automatically routes to the REASONING tier regardless of the weighted score.

2
docs/my-website/docs/proxy/budget_reset_and_tz.md
View file

@ -22,6 +22,8 @@ litellm_settings:
This ensures that all budget resets happen at midnight in your specified timezone rather than in UTC.
If no timezone is specified, UTC will be used by default.
Any valid [IANA timezone string](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) is supported (powered by Python's `zoneinfo` module). DST transitions are handled automatically.
Common timezone values:
- `UTC` - Coordinated Universal Time

1
docs/my-website/docs/proxy/caching.md
View file

@ -340,6 +340,7 @@ litellm_settings:
qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
qdrant_collection_name: test_collection
qdrant_quantization_config: binary
qdrant_semantic_cache_vector_size: 1536 # vector size must match embedding model dimensionality
similarity_threshold: 0.8 # similarity threshold for semantic cache
```

21
docs/my-website/docs/proxy/config_settings.md
View file

@ -73,6 +73,7 @@ litellm_settings:
qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
qdrant_collection_name: test_collection
qdrant_quantization_config: binary
qdrant_semantic_cache_vector_size: 1536 # vector size must match embedding model dimensionality
similarity_threshold: 0.8 # similarity threshold for semantic cache
# Optional - S3 Cache Settings
@ -195,6 +196,7 @@ router_settings:
| disable_end_user_cost_tracking_prometheus_only | boolean | If true, turns off end user cost tracking on prometheus metrics only. |
| key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) |
| disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. |
| use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. |
| disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
| disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
@ -485,6 +487,7 @@ router_settings:
| CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache
| CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service
| COHERE_API_BASE | Base URL for Cohere API. Default is https://api.cohere.com
| COMPETITOR_LLM_TEMPERATURE | Temperature setting for the LLM used in competitor discovery. Default is 0.3
| DATABASE_HOST | Hostname for the database server
| DATABASE_NAME | Name of the database
| DATABASE_PASSWORD | Password for the database user
@ -573,6 +576,8 @@ router_settings:
| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO | Default minimal reasoning effort thinking budget for Gemini 2.5 Pro. Default is 512
| DEFAULT_REDIS_MAJOR_VERSION | Default Redis major version to assume when version cannot be determined. Default is 7
| DEFAULT_REDIS_SYNC_INTERVAL | Default Redis synchronization interval in seconds. Default is 1
| DEFAULT_SEMANTIC_GUARD_EMBEDDING_MODEL | Default embedding model for Semantic Guard (route-matching guardrail). Default is "text-embedding-3-small"
| DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD | Default similarity threshold for Semantic Guard route matching. Default is 0.75
| DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND | Default price per second for Replicate GPU. Default is 0.001400
| DEFAULT_REPLICATE_POLLING_DELAY_SECONDS | Default delay in seconds for Replicate polling. Default is 1
| DEFAULT_REPLICATE_POLLING_RETRIES | Default number of retries for Replicate polling. Default is 5
@ -752,15 +757,18 @@ router_settings:
| LITELLM_ANTHROPIC_BETA_HEADERS_URL | Custom URL for fetching Anthropic beta headers configuration. Default is the GitHub main branch URL
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
| LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker.
| LITELLM_BLOG_POSTS_URL | Custom URL for fetching LiteLLM blog posts JSON. Default is the GitHub main branch URL
| LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
| LITELLM_DEPLOYMENT_ENVIRONMENT | Environment name for the deployment (e.g., "production", "staging"). Used as a fallback when OTEL_ENVIRONMENT_NAME is not set. Sets the `environment` tag in telemetry data
| LITELLM_DETAILED_TIMING | When true, adds detailed per-phase timing headers to responses (`x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms`). Default is false. See [latency overhead docs](../troubleshoot/latency_overhead.md)
| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
| LITELLM_DD_LLM_OBS_PORT | Port for Datadog LLM Observability agent. Default is 8126
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
| LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
| LITELLM_EMAIL | Email associated with LiteLLM account
| LITELLM_FAVICON_URL | Custom URL for the LiteLLM UI favicon. When set, overrides the default favicon
| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES | Maximum retries for parallel requests in LiteLLM
| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM
| LITELLM_DISABLE_LAZY_LOADING | When set to "1", "true", "yes", or "on", disables lazy loading of attributes (currently only affects encoding/tiktoken). This ensures encoding is initialized before VCR starts recording HTTP requests, fixing VCR cassette creation issues. See [issue #18659](https://github.com/BerriAI/litellm/issues/18659)
@ -774,6 +782,7 @@ router_settings:
| LITELLM_KEY_ROTATION_GRACE_PERIOD | Duration to keep old key valid after rotation (e.g. "24h", "2d"). Default is empty (immediate revoke). Used for scheduled rotations and as fallback when not specified in regenerate request.
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False`
| LITELLM_LOCAL_BLOG_POSTS | When set to `True`, uses the local bundled blog posts only, disabling remote fetching from GitHub. Default is `False`
| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
| LITELLM_LOCAL_POLICY_TEMPLATES | When set to "true", uses local backup policy templates instead of fetching from GitHub. Policy templates are fetched from https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json by default, with automatic fallback to local backup on failure
| LITELLM_LOG | Enable detailed logging for LiteLLM
@ -788,6 +797,7 @@ router_settings:
| PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
| PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used.
| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MAX_ITERATIONS_TTL | TTL in seconds for session iteration counters used by the max-iterations limiter. Default is 3600 (1 hour)
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
@ -806,6 +816,8 @@ router_settings:
| LOGGING_WORKER_MAX_QUEUE_SIZE | Maximum size of the logging worker queue. When the queue is full, the worker aggressively clears tasks to make room instead of dropping logs. Default is 50,000
| LOGGING_WORKER_MAX_TIME_PER_COROUTINE | Maximum time in seconds allowed for each coroutine in the logging worker before timing out. Default is 20.0
| LOGGING_WORKER_CLEAR_PERCENTAGE | Percentage of the queue to extract when clearing. Default is 50%
| MAX_BASE64_LENGTH_FOR_LOGGING | Maximum number of base64 characters to keep in logging payloads. Data URIs exceeding this are replaced with a size placeholder. Set to 0 to disable truncation. Default is 64
| MAX_COMPETITOR_NAMES | Maximum number of competitor names allowed in policy template enrichment. Default is 100
| MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000
| MAX_ITERATIONS_TO_CLEAR_QUEUE | Maximum number of iterations to attempt when clearing the logging worker queue during shutdown. Default is 200
| MAX_TIME_TO_CLEAR_QUEUE | Maximum time in seconds to spend clearing the logging worker queue during shutdown. Default is 5.0
@ -828,6 +840,7 @@ router_settings:
| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 50. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
| MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH | Maximum header length for MCP semantic filter tools. Default is 150
| MAX_POLICY_ESTIMATE_IMPACT_ROWS | Maximum number of rows returned when estimating the impact of a policy. Default is 1000
| MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG | Maximum payload size in bytes for full DEBUG serialization. Payloads exceeding this will be truncated in logs. Default is 102400 (100 KB)
| MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001
| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
| MISTRAL_API_BASE | Base URL for Mistral API. Default is https://api.mistral.ai
@ -891,6 +904,13 @@ router_settings:
| POSTHOG_API_URL | Base URL for PostHog API (defaults to https://us.i.posthog.com)
| POSTHOG_MOCK | Enable mock mode for PostHog integration testing. When set to true, intercepts PostHog API calls and returns mock responses without making actual network calls. Default is false
| POSTHOG_MOCK_LATENCY_MS | Mock latency in milliseconds for PostHog API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
| PRISMA_AUTH_RECONNECT_LOCK_TIMEOUT_SECONDS | Lock timeout in seconds for Prisma auth reconnection. Default is 0.1
| PRISMA_AUTH_RECONNECT_TIMEOUT_SECONDS | Timeout in seconds for Prisma auth reconnection attempts. Default is 2.0
| PRISMA_HEALTH_WATCHDOG_ENABLED | Enable the Prisma DB health watchdog that monitors and reconnects on connection loss. Default is true
| PRISMA_HEALTH_WATCHDOG_INTERVAL_SECONDS | Interval in seconds for Prisma health watchdog probes. Default is 30
| PRISMA_HEALTH_WATCHDOG_PROBE_TIMEOUT_SECONDS | Timeout in seconds for each Prisma health probe. Default is 5.0
| PRISMA_RECONNECT_COOLDOWN_SECONDS | Cooldown in seconds between Prisma reconnection attempts. Default is 15
| PRISMA_WATCHDOG_RECONNECT_TIMEOUT_SECONDS | Timeout in seconds for Prisma watchdog-initiated reconnection. Default is 30.0
| PREDIBASE_API_BASE | Base URL for Predibase API
| PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
| PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service
@ -973,6 +993,7 @@ router_settings:
| TOGETHER_AI_EMBEDDING_150_M | Size parameter for Together AI 150M embedding model. Default is 150
| TOGETHER_AI_EMBEDDING_350_M | Size parameter for Together AI 350M embedding model. Default is 350
| TOOL_CHOICE_OBJECT_TOKEN_COUNT | Token count for tool choice objects. Default is 4
| TOOL_POLICY_CACHE_TTL_SECONDS | TTL in seconds for caching tool policy guardrail results. Default is 60
| UI_LOGO_PATH | Path to the logo image used in the UI
| UI_PASSWORD | Password for accessing the UI
| UI_USERNAME | Username for accessing the UI

6
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -161,7 +161,7 @@ Use this when you want non-proxy admins to access `/spend` endpoints
:::info
Schedule a [meeting with us to get your Enterprise License](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
Schedule a [meeting with us to get your Enterprise License](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::
@ -326,6 +326,10 @@ See our [Swagger API](https://litellm-api.up.railway.app/#/Budget%20%26%20Spend%
## Custom Tags
:::tip See Full Request Tags Documentation
For comprehensive documentation on all tag options including `x-litellm-tags` header, request body `tags`, and config-based tags, see the dedicated [Request Tags](./request_tags.md) page.
:::
Requirements:
- Virtual Keys & a database should be set up, see [virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys)

19
docs/my-website/docs/proxy/credential_usage_tracking.md Normal file
View file

@ -0,0 +1,19 @@
# Credential Usage Tracking
When a model is attached to a [reusable credential](./ui_credentials.md), LiteLLM automatically injects the credential name as a tag on every request that uses that model. This means credential-level spend and usage are tracked with zero extra configuration.
## How It Works
When you attach a model to a reusable credential via `litellm_credential_name`, each request routed through that model is tagged `Credential: <name>` (for example, `Credential: xAI`). This tag flows into `DailyTagSpend` and appears in the **Tag** view on the Usage page, where you can filter spend and usage by credential.
If a model has no credential attached, behavior is unchanged—no credential tag is added.
## Viewing Credential Usage
In the Admin UI, go to **Usage → Tag** and look for tags with the `Credential: ` prefix. These represent aggregated spend and token usage across all requests that used that credential.
## Related Documentation
- [Adding LLM Credentials](./ui_credentials.md) - How to create and attach reusable credentials to models
- [Tag Budgets](./tag_budgets.md) - Setting spend limits on tags
- [Tag Routing](./tag_routing.md) - Routing requests based on tags

2
docs/my-website/docs/proxy/email.md
View file

@ -203,7 +203,7 @@ After regenerating the key, the user will receive an email notification with:
:::info
Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/proxy/enterprise.md
View file

@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem';
# ✨ Enterprise Features
:::tip
To get a license, get in touch with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
To get a license, get in touch with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

129
docs/my-website/docs/proxy/forward_client_headers.md
View file

@ -37,11 +37,11 @@ The following rules determine which headers are forwarded (see [`_get_forwardabl
| Rule | Example | Forwarded? |
|---|---|---|
| Headers starting with `x-` | `x-trace-id`, `x-custom-header`, `x-request-source` | ✅ Yes |
| `anthropic-beta` header | `anthropic-beta: prompt-caching-2024-07-31` | ✅ Yes |
| Headers starting with `x-stainless-*` | `x-stainless-lang`, `x-stainless-arch` | ❌ No (causes OpenAI SDK issues) |
| Standard HTTP headers | `Authorization`, `Content-Type`, `Host` | ❌ No |
| Other provider headers | `Accept`, `User-Agent` | ❌ No |
| Headers starting with `x-` | `x-trace-id`, `x-custom-header`, `x-request-source` | Yes |
| `anthropic-beta` header | `anthropic-beta: prompt-caching-2024-07-31` | Yes |
| Headers starting with `x-stainless-*` | `x-stainless-lang`, `x-stainless-arch` | No (causes OpenAI SDK issues) |
| Standard HTTP headers | `Authorization`, `Content-Type`, `Host` | No |
| Other provider headers | `Accept`, `User-Agent` | No |
### Additional Header Mechanisms
@ -61,6 +61,125 @@ general_settings:
forward_client_headers_to_llm_api: true
```
## Forward LLM Provider Authentication Headers
**New in v1.82+**: By default, LiteLLM strips authentication headers like `x-api-key`, `x-goog-api-key`, and `api-key` from client requests for security (these are typically used to authenticate with the proxy itself). However, you can enable forwarding of these LLM provider authentication headers to allow **Bring Your Own Key (BYOK)** scenarios where clients send their own API keys to the LLM provider.
### Configuration
Add `forward_llm_provider_auth_headers: true` to your `general_settings`:
```yaml
general_settings:
forward_client_headers_to_llm_api: true
forward_llm_provider_auth_headers: true # 👈 Enable BYOK
```
### Which Headers Are Forwarded
When `forward_llm_provider_auth_headers: true`, the following LLM provider authentication headers are preserved and forwarded:
| Header | Provider | Example |
|--------|----------|---------|
| `x-api-key` | Anthropic, Azure AI, Databricks | `x-api-key: sk-ant-api03-...` |
| `x-goog-api-key` | Google AI Studio | `x-goog-api-key: AIza...` |
| `api-key` | Azure OpenAI | `api-key: your-azure-key` |
| `ocp-apim-subscription-key` | Azure APIM | `ocp-apim-subscription-key: your-key` |
:::warning Important Security Note
The proxy's `Authorization` header (used for proxy authentication) is **never** forwarded to LLM providers, even with this setting enabled. This ensures your proxy authentication remains secure.
:::
### Use Case: Client-Side API Keys (BYOK)
This feature enables scenarios where:
1. **Clients bring their own LLM provider API keys** instead of using keys configured in the proxy
2. **Multi-tenant applications** where each tenant has their own Anthropic/OpenAI account
3. **Development environments** where developers use their personal API keys through a shared proxy
#### Example: Anthropic BYOK
```yaml
# proxy_config.yaml
model_list:
- model_name: claude-sonnet-4
litellm_params:
model: anthropic/claude-sonnet-4-20250514
# No api_key configured! Will use client's key
general_settings:
forward_client_headers_to_llm_api: true
forward_llm_provider_auth_headers: true # Enable BYOK
```
Client request:
```bash
curl -X POST "http://localhost:4000/v1/messages" \
-H "Authorization: Bearer sk-proxy-auth-123" \ # Proxy authentication (stripped)
-H "x-api-key: sk-ant-api03-YOUR-KEY..." \ # Client's Anthropic key (forwarded!)
-H "Content-Type: application/json" \
-d '{
"model": "claude-sonnet-4",
"messages": [{"role": "user", "content": "Hello"}],
"max_tokens": 100
}'
```
#### Example: Google AI Studio BYOK
```yaml
model_list:
- model_name: gemini-pro
litellm_params:
model: gemini/gemini-1.5-pro
# No api_key configured
general_settings:
forward_client_headers_to_llm_api: true
forward_llm_provider_auth_headers: true
```
Client request:
```bash
curl -X POST "http://localhost:4000/v1/chat/completions" \
-H "Authorization: Bearer sk-proxy-auth-123" \
-H "x-goog-api-key: AIza..." \
-d '{
"model": "gemini-pro",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
### Security Considerations
**When to Use This Feature:**
- Internal tools where you trust all clients
- Development/testing environments
- Multi-tenant apps with proper client authentication
- Scenarios where you want clients to use their own API keys
**When NOT to Use:**
- Public APIs where you don't trust all clients
- When you want centralized billing/cost control
- When you need to enforce rate limits at the proxy level
### Backward Compatibility
For backward compatibility, if you have `forward_client_headers_to_llm_api: true` but don't explicitly set `forward_llm_provider_auth_headers`, the behavior is:
- **Default**: LLM provider auth headers are **NOT** forwarded (safe default)
- **Explicit `true`**: LLM provider auth headers **ARE** forwarded (BYOK enabled)
```yaml
# Safe default - auth headers NOT forwarded
general_settings:
forward_client_headers_to_llm_api: true
# BYOK enabled - auth headers ARE forwarded
general_settings:
forward_client_headers_to_llm_api: true
forward_llm_provider_auth_headers: true # 👈 Opt-in required
```
## Enable for a Model Group
Add the `forward_client_headers_to_llm_api` setting under `model_group_settings` in your configuration:

2
docs/my-website/docs/proxy/guardrails/aporia_api.md
View file

@ -139,7 +139,7 @@ curl -i http://localhost:4000/v1/chat/completions \
:::info
✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/proxy/guardrails/custom_guardrail.md
View file

@ -409,7 +409,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \
:::info
✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/proxy/guardrails/guardrails_ai.md
View file

@ -59,7 +59,7 @@ curl -i http://localhost:4000/v1/chat/completions \
:::info
✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/proxy/guardrails/lakera_ai.md
View file

@ -4,6 +4,8 @@ import TabItem from '@theme/TabItem';
# Lakera AI
**Supported endpoints:** The Lakera v2 integration only supports the **chat completions** endpoint (`/v1/chat/completions`). It is not supported for the Responses API, `/v1/messages`, MCP, A2A, or other proxy endpoints.
## Quick Start
### 1. Define Guardrails on your LiteLLM config.yaml

102
docs/my-website/docs/proxy/guardrails/noma_security.md
View file

@ -6,6 +6,108 @@ import TabItem from '@theme/TabItem';
Use [Noma Security](https://noma.security/) to protect your LLM applications with comprehensive AI content moderation and safety guardrails.
:::warning Deprecated: `guardrail: noma` (Legacy)
`guardrail: noma` is deprecated and users should migrate to `guardrail: noma_v2`.
The legacy `guardrail: noma` API will no longer be supported after March 31, 2026.
For easier migration of existing integrations, keep `guardrail: noma` and set `use_v2: true`.
With `use_v2: true`, requests route to `noma_v2`; `monitor_mode` and `block_failures` still apply, while `anonymize_input` is ignored.
:::
## Noma v2 guardrails (Recommended)
### Quick Start
```yaml showLineNumbers title="litellm config.yaml"
guardrails:
- guardrail_name: "noma-v2-guard"
litellm_params:
guardrail: noma_v2
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
api_base: os.environ/NOMA_API_BASE
```
If you want to migrate gradually without changing guardrail names yet:
```yaml showLineNumbers title="litellm config.yaml"
guardrails:
- guardrail_name: "noma-guard"
litellm_params:
guardrail: noma
use_v2: true
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
api_base: os.environ/NOMA_API_BASE
```
### Supported Params
- **`guardrail`**: Use `noma_v2` (recommended), or `noma` with `use_v2: true` for migration
- **`mode`**: `pre_call`, `post_call`, `during_call`, `pre_mcp_call`, `during_mcp_call`
- **`api_key`**: Noma API key (required for Noma SaaS, optional for self-managed deployments)
- **`api_base`**: Noma API base URL (defaults to `https://api.noma.security/`)
- **`application_id`**: Application identifier. If omitted, v2 checks dynamic `extra_body.application_id`, then configured/env `application_id`; otherwise it is omitted.
- **`monitor_mode`**: If `true`, runs in monitor-only mode without blocking (defaults to `false`)
- **`block_failures`**: If `true`, fail-closed on guardrail technical failures (defaults to `true`)
- **`use_v2`**: Migration toggle when `guardrail: noma` is used
### Environment Variables
```shell
export NOMA_API_KEY="your-api-key-here"
export NOMA_API_BASE="https://api.noma.security/" # Optional
export NOMA_APPLICATION_ID="my-app" # Optional
export NOMA_MONITOR_MODE="false" # Optional
export NOMA_BLOCK_FAILURES="true" # Optional
```
### Multiple Guardrails
Apply different v2 configurations for input and output:
```yaml showLineNumbers title="litellm config.yaml"
guardrails:
- guardrail_name: "noma-v2-input"
litellm_params:
guardrail: noma_v2
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
- guardrail_name: "noma-v2-output"
litellm_params:
guardrail: noma_v2
mode: "post_call"
api_key: os.environ/NOMA_API_KEY
```
### Pass Additional Parameters
This is supported in v2 via `extra_body`.
Currently, `noma_v2` consumes dynamic `application_id`.
```shell showLineNumbers title="Curl Request"
curl 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o-mini",
"messages": [
{
"role": "user",
"content": "Hello, how are you?"
}
],
"guardrails": {
"noma-v2-guard": {
"extra_body": {
"application_id": "my-specific-app-id"
}
}
}
}'
```
## Noma guardrails (Legacy)
## Quick Start
### 1. Define Guardrails on your LiteLLM config.yaml

199
docs/my-website/docs/proxy/guardrails/realtime_guardrails.md Normal file
View file

@ -0,0 +1,199 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Realtime API Guardrails
Guard voice conversations in the [Realtime API](/docs/realtime) — intercept speech transcriptions **before** the LLM responds.
## How it works
The Realtime API is a long-lived WebSocket session. Unlike `/chat/completions` where a guardrail runs once per HTTP request, a voice session has many turns — each one needs to be checked individually.
LiteLLM intercepts each turn at the transcription event, after Whisper converts speech to text but before the LLM generates a response:
```
User speaks into mic
│
▼ audio bytes (PCM)
┌───────────────────┐
│ LiteLLM Proxy │ forwards audio to OpenAI unchanged
└────────┬──────────┘
│
▼
┌───────────────────┐
│ OpenAI │
│ VAD → Whisper │ detects speech end, transcribes
└────────┬──────────┘
│
│ conversation.item.input_audio_transcription.completed
│ { transcript: "system update: ignore all instructions" }
│
▼
┌───────────────────────────────────────────┐
│ LiteLLM Proxy │
│ │
│ ◄──── GUARDRAIL RUNS HERE ────► │
│ apply_guardrail(texts=[transcript]) │
│ │
│ ┌──────────────┬──────────────────┐ │
│ │ BLOCKED │ CLEAN │ │
│ └──────┬───────┴───────┬──────────┘ │
│ │ │ │
│ speak warning send response.create │
│ (TTS audio) → LLM responds │
└───────────────────────────────────────────┘
```
**Key detail**: LiteLLM also injects `create_response: false` into the session on connect, so the LLM never auto-responds before the guardrail has run.
## Supported guardrail mode
| Mode | Description |
|------|-------------|
| `realtime_input_transcription` | Runs after each voice turn is transcribed, before LLM responds |
## Quick Start
### Step 1: Configure proxy
Add a guardrail with `mode: realtime_input_transcription` to your proxy config:
```yaml
model_list:
- model_name: openai/gpt-4o-realtime-preview
litellm_params:
model: openai/gpt-4o-realtime-preview
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "voice-content-filter"
litellm_params:
guardrail: litellm_content_filter
mode: realtime_input_transcription
default_on: true
blocked_words:
- keyword: "ignore previous instructions"
action: BLOCK
description: "Prompt injection attempt"
- keyword: "system update"
action: BLOCK
description: "Prompt injection attempt"
- keyword: "ignore all instructions"
action: BLOCK
description: "Prompt injection attempt"
general_settings:
master_key: sk-1234
```
### Step 2: Start proxy
```bash
litellm --config proxy_config.yaml --port 4000
```
### Step 3: Connect a Realtime client
Connect your client to the proxy instead of directly to OpenAI:
<Tabs>
<TabItem value="js" label="JavaScript">
```javascript
const ws = new WebSocket(
"ws://localhost:4000/v1/realtime?model=openai/gpt-4o-realtime-preview",
[],
{ headers: { Authorization: "Bearer sk-1234" } }
)
ws.onopen = () => {
ws.send(JSON.stringify({
type: "session.update",
session: {
modalities: ["audio", "text"],
input_audio_transcription: { model: "whisper-1" },
turn_detection: { type: "server_vad" },
},
}))
}
ws.onmessage = (e) => {
const event = JSON.parse(e.data)
if (event.type === "response.audio.delta") {
// play audio...
}
}
```
</TabItem>
<TabItem value="python" label="Python">
```python
import asyncio
import json
import websockets
async def main():
async with websockets.connect(
"ws://localhost:4000/v1/realtime?model=openai/gpt-4o-realtime-preview",
additional_headers={"Authorization": "Bearer sk-1234"},
) as ws:
await ws.recv() # session.created
await ws.send(json.dumps({
"type": "session.update",
"session": {
"modalities": ["audio", "text"],
"input_audio_transcription": {"model": "whisper-1"},
"turn_detection": {"type": "server_vad"},
},
}))
async for raw in ws:
event = json.loads(raw)
print(event["type"])
asyncio.run(main())
```
</TabItem>
</Tabs>
### What happens when a turn is blocked
When the guardrail fires, the proxy:
1. Sends `response.cancel` to kill any in-flight LLM response
2. Sends `response.create` with the block message as forced instructions
3. OpenAI's TTS **speaks the warning** back to the user — e.g. *"Content blocked: keyword 'system update' detected (Prompt injection attempt)"*
The LLM never processes the injected instruction.
## Using with any guardrail provider
`realtime_input_transcription` mode works with any guardrail that implements `apply_guardrail`. Just swap `litellm_content_filter` for your provider:
```yaml
guardrails:
- guardrail_name: "voice-lakera"
litellm_params:
guardrail: lakera_ai
mode: realtime_input_transcription
default_on: true
api_key: os.environ/LAKERA_API_KEY
```
## Per-key guardrail control
To enable realtime guardrails only for specific API keys, set `default_on: false` and pass the guardrail name in the request metadata:
```yaml
guardrails:
- guardrail_name: "voice-content-filter"
litellm_params:
guardrail: litellm_content_filter
mode: realtime_input_transcription
default_on: false # off by default
```
Then the client opts in per-connection by passing it in the initial metadata (enterprise feature).

2
docs/my-website/docs/proxy/ip_address.md
View file

@ -3,7 +3,7 @@
:::info
You need a LiteLLM License to unlock this feature. [Grab time](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat), to get one today!
You need a LiteLLM License to unlock this feature. [Grab time](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions), to get one today!
:::

6
docs/my-website/docs/proxy/logging.md
View file

@ -1109,7 +1109,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage?
:::info
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::
@ -1194,7 +1194,7 @@ Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.goog
:::info
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::
@ -1497,7 +1497,7 @@ Log LLM Logs to [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azur
:::info
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/proxy/multiple_admins.md
View file

@ -20,7 +20,7 @@ LiteLLM tracks changes to the following entities and actions:
:::tip
Requires Enterprise License, Get in touch with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
Requires Enterprise License, Get in touch with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/proxy/oauth2.md
View file

@ -4,7 +4,7 @@ Use this if you want to use an Oauth2.0 token to make `/chat`, `/embeddings` req
:::info
This is an Enterprise Feature - [get in touch with us if you want a free trial to test if this feature meets your needs]((https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat))
This is an Enterprise Feature - [get in touch with us if you want a free trial to test if this feature meets your needs]((https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions))
:::

2
docs/my-website/docs/proxy/prod.md
View file

@ -47,7 +47,7 @@ export LITELLM_LOG="ERROR"
:::info
Need Help or want dedicated support ? Talk to a founder [here]: (https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
Need Help or want dedicated support ? Talk to a founder [here]: (https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

26
docs/my-website/docs/proxy/prometheus.md
View file

@ -122,7 +122,7 @@ Use this to track overall LiteLLM Proxy usage.
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "user_email", "exception_status", "exception_class", "route", "model_id"` |
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"` |
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route", "model_id"`. Optionally includes `"stream"` — see [Emit Stream Label](#emit-stream-label). |
### Callback Logging Metrics
@ -214,9 +214,31 @@ litellm_settings:
```
### Emit Stream Label
Add a `stream` label to `litellm_proxy_total_requests_metric` to split requests by streaming vs. non-streaming. Disabled by default.
```yaml title="config.yaml"
litellm_settings:
callbacks: ["prometheus"]
prometheus_emit_stream_label: true
```
When enabled, `litellm_proxy_total_requests_metric` gains a `stream` label with values `"True"`, `"False"`, or `"None"`.
```
litellm_proxy_total_requests_metric{..., stream="True"} 42
litellm_proxy_total_requests_metric{..., stream="False"} 100
```
:::note
This label is opt-in because adding a new label to an existing metric changes its cardinality and breaks existing Prometheus queries / Grafana dashboards that target this metric. Enable it only on fresh deployments or when you are ready to update your dashboards.
:::
## [BETA] Custom Metrics
Track custom metrics on prometheus on all events mentioned above.
Track custom metrics on prometheus on all events mentioned above.
### Custom Metadata Labels

2
docs/my-website/docs/proxy/public_routes.md
View file

@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem';
:::info
Requires a LiteLLM Enterprise License. [Get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat).
Requires a LiteLLM Enterprise License. [Get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions).
:::

128
docs/my-website/docs/proxy/request_tags.md
View file

@ -1,9 +1,16 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Request Tags for Spend Tracking
Add tags to model deployments to track spend by environment, AWS account, or any custom label.
Tags appear in the `request_tags` field of LiteLLM spend logs.
:::info Requirements
Virtual Keys & a database should be set up. See [Virtual Keys Setup](./virtual_keys.md).
:::
## Config Setup
Set tags on model deployments in `config.yaml`:
@ -27,7 +34,9 @@ model_list:
## Make Request
Requests just specify the model - tags are automatically applied:
### Option 1: Use Config Tags (Automatic)
Requests just specify the model - tags are automatically applied from config:
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
@ -39,6 +48,120 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
}'
```
### Option 2: Use `x-litellm-tags` Header
Pass tags dynamically via the `x-litellm-tags` header as a comma-separated string:
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-H 'x-litellm-tags: team-api,production,us-east-1' \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
Format: Comma-separated string (spaces are automatically trimmed): `"tag1,tag2,tag3"`
### Option 3: Use Request Body `tags`
Pass tags directly in the request body. Both formats are supported:
<Tabs>
<TabItem value="direct" label="Direct tags Field">
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
"tags": ["team-api", "production", "us-east-1"]
}'
```
</TabItem>
<TabItem value="metadata" label="Metadata Nested">
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
"metadata": {
"tags": ["team-api", "production", "us-east-1"]
}
}'
```
</TabItem>
</Tabs>
The `tags` field must be an array of strings.
:::info
When tags are provided via header or request body, they override any tags configured in the model deployment. If both header and body tags are provided, body tags take precedence.
:::
## Set Tags on Keys or Teams
You can also set default tags at the API key or team level:
<Tabs>
<TabItem value="key" label="Set on Key">
```bash
curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {
"tags": ["customer-acme", "tier-premium"]
}
}'
```
</TabItem>
<TabItem value="team" label="Set on Team">
```bash
curl -L -X POST 'http://0.0.0.0:4000/team/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {
"tags": ["team-engineering", "department-ai"]
}
}'
```
</TabItem>
</Tabs>
## Advanced: Custom Header Tracking
Track spend using any custom header by adding it to your config:
```yaml
litellm_settings:
extra_spend_tag_headers:
- "x-custom-header"
- "x-customer-id"
```
**Disable User-Agent tracking:**
```yaml
litellm_settings:
disable_add_user_agent_to_request_tags: true
```
## Spend Logs
The tag from the model config appears in `LiteLLM_SpendLogs`:
@ -54,5 +177,6 @@ The tag from the model config appears in `LiteLLM_SpendLogs`:
## Related
- [Spend Tracking Overview](cost_tracking.md)
- [Spend Tracking Overview](cost_tracking.md) - Complete tutorial on tracking spend with tags
- [Tag Budgets](tag_budgets.md) - Set budget limits per tag
- [Virtual Keys Setup](virtual_keys.md) - Required for tag tracking

2
docs/my-website/docs/proxy/tag_routing.md
View file

@ -215,7 +215,7 @@ LiteLLM Proxy supports team-based tag routing, allowing you to associate specifi
:::info
This is an enterprise feature, [Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
This is an enterprise feature, [Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

4
docs/my-website/docs/proxy/team_logging.md
View file

@ -26,7 +26,7 @@ Team 3 -> Disabled Logging (for GDPR compliance)
:::info
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::
@ -248,7 +248,7 @@ Use the `/key/generate` or `/key/update` endpoints to add logging callbacks to a
:::info
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/proxy/team_model_add.md
View file

@ -5,7 +5,7 @@
This is an Enterprise feature.
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/proxy/token_auth.md
View file

@ -11,7 +11,7 @@ Use JWT's to auth admins / users / projects into the proxy.
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

4
docs/my-website/docs/proxy/ui_credentials.md
View file

@ -46,6 +46,10 @@ Go to Add Model -> Existing Credentials -> Select your credential in the dropdow
<Image img={require('../../img/use_model_cred.png')} />
## Usage Tracking
Models attached to a reusable credential are automatically tracked in the Usage page. Each request is tagged `Credential: <name>` and appears in the **Tag** view, so you can filter spend and usage by credential without any extra configuration. See [Credential Usage Tracking](./credential_usage_tracking.md) for details.
## Frequently Asked Questions

92
docs/my-website/docs/proxy/ui_store_model_db_setting.md Normal file
View file

@ -0,0 +1,92 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Store Model in DB Settings
Enable or disable storing model definitions in the database directly from the Admin UI—no config file edits or proxy restart required. This is especially useful for cloud deployments where updating the config is difficult or requires a long release process.
## Overview
Previously, the `store_model_in_db` setting had to be configured in `proxy_config.yaml` under `general_settings`. Changing it required editing the config and restarting the proxy, which was problematic for cloud users who don't have direct access to the config file or who want to avoid the downtime caused by restarts.
<Image img={require('../../img/ui_store_model_in_db.png')} />
**Store Model in DB Settings** lets you:
- **Enable or disable storing models in the database** – Control whether model definitions are cached in your database (useful for reducing config file size and improving scalability)
- **Apply changes immediately** – No proxy restart needed; settings take effect for new model operations as soon as you save
:::warning UI overrides config
Settings changed in the UI **override** the values in your config file. For example, if `store_model_in_db` is set to `false` in `general_settings`, enabling it in the UI will still persist model definitions to the database. Use the UI when you want runtime control without redeploying.
:::
## How Store Model in DB Works
When `store_model_in_db` is enabled, the LiteLLM proxy stores model definitions in the database instead of relying solely on your `proxy_config.yaml`. This provides several benefits:
- **Reduced config size** – Move model definitions out of YAML for easier maintenance
- **Scalability** – Database storage scales better than large YAML files
- **Dynamic updates** – Models can be added or updated without editing config files
- **Persistence** – Model definitions persist across proxy instances and restarts
The setting applies to all new model operations from the moment you save it.
## How to Configure Store Model in DB in the UI
### 1. Access Models + Endpoints Settings
Navigate to the Admin UI (e.g. `http://localhost:4000/ui` or your `PROXY_BASE_URL/ui`) and go to the **Models + Endpoints** page.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-22/55bc71f5-730f-4b2c-8539-8a4f46b8bd10/ascreenshot_0f7ba8f1c2694e94938996fd1b4adfcc_text_export.jpeg)
### 2. Open Settings
Click **Models + Endpoints** from the navigation menu.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-22/55bc71f5-730f-4b2c-8539-8a4f46b8bd10/ascreenshot_fc2b9e4812a9480087f4eb350fa0a792_text_export.jpeg)
### 3. Click the Settings Icon
Look for the settings (gear) icon on the Models + Endpoints page to open the configuration panel.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-22/7b394364-c281-4db8-8cad-ee322c76c935/ascreenshot_d7c8a6b234bc4e4d92aa7f09aefb13d3_text_export.jpeg)
### 4. Enable or Disable Store Model in DB
Toggle the **Store Model in DB** setting based on your preference:
- **Enabled**: Model definitions will be stored in the database
- **Disabled**: Models are read from the config file only
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-22/54a263ec-ad67-4b16-ba9f-2be57c3e4cb8/ascreenshot_501abda2a6c847f79d085efce814265d_text_export.jpeg)
### 5. Save Settings
Click **Save Settings** to apply the change. No proxy restart is required; the new setting takes effect immediately for subsequent model operations.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-22/7d13559a-d4e4-41f7-993b-cb20fbfa1f6e/ascreenshot_3245f3c5bd0d43cb96c5f5ff0ccb461d_text_export.jpeg)
## Use Cases
### Cloud and Managed Deployments
When the proxy runs in a managed or cloud environment, config may be in a separate repo, require a long release cycle, or be controlled by another team. Using the UI lets you change the `store_model_in_db` setting without going through a deployment process.
### Reducing Configuration Complexity
For large deployments with hundreds of models, storing model definitions in the database reduces the size and complexity of your `proxy_config.yaml`, making it easier to maintain and version control.
### Dynamic Model Management
Enable `store_model_in_db` to support dynamic model additions and updates without editing your config file. Teams can manage models through the UI or API without needing to redeploy the proxy.
### Zero-Downtime Updates
Change the setting from the UI and have it take effect immediately—perfect for production environments where downtime must be minimized.
## Related Documentation
- [Admin UI Overview](./ui_overview.md) – General guide to the LiteLLM Admin UI
- [Models and Endpoints](./models_and_endpoints.md) – Managing models and API endpoints
- [Config Settings](./config_settings.md) – `store_model_in_db` in `general_settings`

85
docs/my-website/docs/realtime.md
View file

@ -85,7 +85,7 @@ const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio";
// const url = "wss://my-endpoint-sweden-berri992.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview";
const ws = new WebSocket(url, {
headers: {
"api-key": `f28ab7b695af4154bc53498e5bdccb07`,
"api-key": `sk-1234`,
"OpenAI-Beta": "realtime=v1",
},
});
@ -110,7 +110,88 @@ ws.on("error", function handleError(error) {
});
```
## Logging
## Guardrails
You can apply [LiteLLM guardrails](https://docs.litellm.ai/docs/proxy/guardrails/quick_start) to realtime sessions.
### Set guardrails on a key or team
The easiest production setup — attach guardrails to a virtual key or team so they always apply automatically, without any client-side changes.
See [Virtual Keys → Guardrails](https://docs.litellm.ai/docs/proxy/virtual_keys#guardrails) and [Teams → Guardrails](https://docs.litellm.ai/docs/proxy/team_budgets).
### Pass guardrails dynamically (easy testing)
Pass `guardrails` as a query param when opening the WebSocket.
Useful for testing guardrails without modifying key/team config.
```js
// node test.js
const WebSocket = require("ws");
const guardrails = ["your-guardrail-name"]; // comma-separated list
const url = `ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio&guardrails=${guardrails.join(",")}`;
const ws = new WebSocket(url, {
headers: {
"Authorization": "Bearer sk-1234",
},
});
ws.on("open", function open() {
console.log("Connected — guardrails active:", guardrails);
});
ws.on("message", function incoming(message) {
const data = JSON.parse(message);
if (data.type === "error") {
// Guardrail block is sent as an error event before the connection closes
console.error("Guardrail error:", data.error.message);
}
});
ws.on("close", function close(code, reason) {
console.log("Closed:", code, reason.toString());
// code 1011 = blocked by guardrail at pre_call
});
```
Or with Python:
```python
import asyncio
import websockets
async def main():
url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio&guardrails=your-guardrail-name"
async with websockets.connect(
url,
additional_headers={"Authorization": "Bearer sk-1234"},
) as ws:
print("Connected — guardrail active")
async for msg in ws:
import json
data = json.loads(msg)
if data["type"] == "error":
print("Guardrail blocked:", data["error"]["message"])
break
asyncio.run(main())
```
When a guardrail blocks the request, the proxy sends an `error` event over the WebSocket and then closes the connection:
```json
{
"type": "error",
"error": {
"type": "guardrail_error",
"message": "Guardrail blocked this request: <reason>"
}
}
```
## Logging
To prevent requests from being dropped, by default LiteLLM just logs these event types:

19
docs/my-website/docs/reasoning_content.md
View file

@ -642,6 +642,25 @@ model_list:
model: openai/responses/gpt-5-mini
```
**Per-model configuration** (recommended when using Open WebUI or clients that cannot set `extra_body`):
```yaml
model_list:
- model_name: gpt-5.1
litellm_params:
model: openai/gpt-5.1
# String format - uses reasoning_auto_summary for summary when set
reasoning_effort: "high"
model_info:
mode: responses # if using Responses API bridge
- model_name: gpt-5.1-with-summary
litellm_params:
model: openai/gpt-5.1
# Dict format - explicit control over effort and summary
reasoning_effort: {"effort": "high", "summary": "detailed"}
```
</TabItem>
</Tabs>

6
docs/my-website/docs/response_api.md
View file

@ -887,7 +887,8 @@ router = litellm.Router(
# `responses_api_deployment_check` ensures Requests with `previous_response_id`
# are routed to the same deployment. `deployment_affinity` adds sticky sessions
# for requests without `previous_response_id` (useful for implicit caching).
optional_pre_call_checks=["responses_api_deployment_check", "deployment_affinity"],
# `session_affinity` adds sticky sessions based on `session_id` metadata.
optional_pre_call_checks=["responses_api_deployment_check", "deployment_affinity", "session_affinity"],
# Optional (default is 3600 seconds / 1 hour)
deployment_affinity_ttl_seconds=3600,
)
@ -919,10 +920,12 @@ follow_up = await router.aresponses(
To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks` in your proxy config.yaml.
- `responses_api_deployment_check`: high priority routing when `previous_response_id` is provided
- `session_affinity`: sticky sessions based on session id (takes priority over `deployment_affinity`)
- `deployment_affinity`: sticky sessions based on user key (applies even without `previous_response_id`)
Notes:
- User-key affinity is keyed on `metadata.user_api_key_hash` (the API key hash). The OpenAI `user` request parameter is an end-user identifier and is intentionally not used for deployment affinity.
- Session-ID affinity is keyed on `metadata.session_id`. For proxy requests, this can be passed via the `x-litellm-session-id` HTTP header. For Python SDK requests, you can pass it via `litellm_metadata={"session_id": "value"}` in request args.
- `user_api_key_hash` is already SHA-256, and is used as-is (no double hashing).
- Affinity is scoped by a stable model identifier (the model-map key, e.g. `model_map_information.model_map_key`) so model aliases map to the same stickiness bucket.
- The mapping TTL is controlled by `deployment_affinity_ttl_seconds` (configured on Router init / proxy startup).
@ -945,6 +948,7 @@ model_list:
router_settings:
optional_pre_call_checks:
- responses_api_deployment_check
- session_affinity
- deployment_affinity
# Optional (default is 3600 seconds / 1 hour)
deployment_affinity_ttl_seconds: 3600

2
docs/my-website/docs/secret.md
View file

@ -6,7 +6,7 @@
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/secret_managers/aws_kms.md
View file

@ -6,7 +6,7 @@
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/secret_managers/aws_secret_manager.md
View file

@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem';
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/secret_managers/azure_key_vault.md
View file

@ -6,7 +6,7 @@
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/secret_managers/cyberark.md
View file

@ -8,7 +8,7 @@ import Image from '@theme/IdealImage';
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/secret_managers/google_kms.md
View file

@ -6,7 +6,7 @@
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/secret_managers/google_secret_manager.md
View file

@ -6,7 +6,7 @@
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/secret_managers/hashicorp_vault.md
View file

@ -8,7 +8,7 @@ import Image from '@theme/IdealImage';
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

2
docs/my-website/docs/secret_managers/overview.md
View file

@ -8,7 +8,7 @@ import Image from '@theme/IdealImage';
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
[Contact us here to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
:::

90
docs/my-website/docs/troubleshoot/latency_overhead.md Normal file
View file

@ -0,0 +1,90 @@
# Latency Overhead Troubleshooting
Use this guide when you see unexpected latency overhead between LiteLLM proxy and the LLM provider.
## Quick Checklist
1. **Collect the `x-litellm-overhead-duration-ms` response header** — this tells you LiteLLM's total overhead on every request. Start here.
2. **Is DEBUG logging enabled?** This is the #1 cause of latency with large payloads.
3. **Are you sending large base64 payloads?** (images, PDFs) — see [Large Payload Overhead](#large-payload-overhead).
4. **Enable detailed timing headers** to pinpoint where time is spent.
## Diagnostic Headers
### `x-litellm-overhead-duration-ms` (always on)
Every response from LiteLLM includes this header. It shows the total latency overhead in milliseconds added by LiteLLM proxy (i.e. total response time minus the LLM API call time). Collect this on every request to understand your baseline overhead.
```bash
curl -s -D - http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-..." \
-d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}' \
2>&1 | grep x-litellm-overhead-duration-ms
```
### `x-litellm-callback-duration-ms` (always on)
Shows time spent building callback/logging payloads (ms). If this is high (>100ms), your payloads may be too large for efficient logging.
```bash
curl -s -D - http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-..." \
-d '{"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}' \
2>&1 | grep x-litellm
```
### Detailed Timing Breakdown (opt-in)
Set `LITELLM_DETAILED_TIMING=true` to get per-phase timing in response headers:
| Header | What it measures |
|--------|-----------------|
| `x-litellm-timing-pre-processing-ms` | Auth, routing, request processing (before LLM call) |
| `x-litellm-timing-llm-api-ms` | Actual LLM API call duration |
| `x-litellm-timing-post-processing-ms` | Response processing (after LLM returns) |
| `x-litellm-timing-message-copy-ms` | Message copy time in logging layer |
```bash
# Enable detailed timing
export LITELLM_DETAILED_TIMING=true
```
## Large Payload Overhead
When sending large payloads (>1MB, e.g. base64-encoded images/PDFs), three things can add overhead:
### 1. DEBUG Logging (most common)
When `LITELLM_LOG=DEBUG` or `set_verbose=True` is enabled, every request payload is serialized with `json.dumps(indent=4)` synchronously. For a 2MB+ payload, this alone can take **2-5 seconds**.
**Fix:** Don't use DEBUG logging in production. Use `INFO` level instead:
```bash
export LITELLM_LOG=INFO
```
If you need DEBUG logging but have large payloads, you can increase the size threshold for full payload logging:
```bash
# Only fully serialize payloads under 100KB for DEBUG logs (default)
export MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG=102400
```
### 2. Base64 in Logging Payloads
Callback payloads (sent to Langfuse, etc.) include message content. Large base64 strings are automatically truncated to size placeholders in logging payloads.
You can control the truncation threshold:
```bash
# Max base64 characters before truncation (default: 64)
export MAX_BASE64_LENGTH_FOR_LOGGING=64
```
## Environment Variables Reference
| Variable | Default | Description |
|----------|---------|-------------|
| `LITELLM_DETAILED_TIMING` | `false` | Enable per-phase timing headers |
| `MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG` | `102400` | Max payload bytes for full DEBUG serialization |
| `MAX_BASE64_LENGTH_FOR_LOGGING` | `64` | Max base64 chars before truncation in logging |

8
docs/my-website/docs/troubleshoot/prisma_migrations.md
View file

@ -2,6 +2,8 @@
Common Prisma migration issues encountered when upgrading or downgrading LiteLLM proxy versions, and how to fix them.
For a full guide on safely reverting your LiteLLM version, see the **[Safe Rollback Guide](rollback)**.
## How Prisma Migrations Work in LiteLLM
- LiteLLM uses [Prisma](https://www.prisma.io/) to manage its PostgreSQL database schema.
@ -46,6 +48,8 @@ After deleting the entry, restart LiteLLM — it will re-apply the migration on
If deleting the migration entry and restarting doesn't resolve the issue, sync the schema directly:
> **Warning:** `prisma db push` can cause **data loss** if the Prisma schema removes columns or tables that exist in your database. Only use this as a last resort and ensure you have a database backup first.
```bash
DATABASE_URL="<your_database_url>" prisma db push
```
@ -76,7 +80,7 @@ DELETE FROM "_prisma_migrations"
WHERE migration_name = '<failed_migration_name>';
```
3. If that doesn't work, use `prisma db push`:
3. If that doesn't work, use `prisma db push` (see [warning above](#step-2--if-that-doesnt-work-use-prisma-db-push) — back up your database first):
```bash
DATABASE_URL="<your_database_url>" prisma db push
@ -106,7 +110,7 @@ LIMIT 20;
3. Restart LiteLLM to re-run migrations.
4. If that doesn't work, use `prisma db push`:
4. If that doesn't work, use `prisma db push` (see [warning above](#step-2--if-that-doesnt-work-use-prisma-db-push) — back up your database first):
```bash
DATABASE_URL="<your_database_url>" prisma db push

115
docs/my-website/docs/troubleshoot/rollback.md Normal file
View file

@ -0,0 +1,115 @@
# Safe Rollback Guide
This guide outlines the process for safely rolling back a LiteLLM Proxy deployment to a previous version.
We recommend rolling back to the previous [stable release](https://github.com/BerriAI/litellm/releases). Stable releases come out every week and follow the `main-v<VERSION>-stable` tag convention (e.g., `main-v1.77.2-stable`).
## 1. Determine Rollback Scope
Before proceeding, identify why you are rolling back:
- **Application Logic Error**: Reverting code changes but keeping the database schema.
- **Database Migration Failure**: Reverting changes that included database schema updates.
- **Performance Regression**: Reverting to a known stable version.
## 2. Back Up the Database
> **Always back up before rolling back.** Before making any changes, take a database snapshot or dump. This is your safety net if something goes wrong during the rollback.
```bash
# PostgreSQL example
pg_dump -h <host> -U <user> -d <database> -F c -f litellm_backup_$(date +%Y%m%d_%H%M%S).dump
```
If you are on a managed database (e.g., AWS RDS, GCP Cloud SQL), create a snapshot through your cloud console instead.
## 3. Pre-Rollback Checks
Before reverting, review these items:
- **`LITELLM_SALT_KEY`**: Do **not** change this value during rollback. It is used to encrypt/decrypt your LLM API Key credentials stored in the database. Changing it will make existing credentials unreadable. See [Best Practices for Production](../proxy/prod#8-set-litellm-salt-key).
- **`config.yaml`**: If you added settings specific to the newer version, the older version may not recognize them. Review your config and remove or comment out any settings that were introduced in the version you are rolling back from.
- **`DISABLE_SCHEMA_UPDATE`**: If you use the [Helm PreSync hook for migrations](../proxy/prod#7-use-helm-presync-hook-for-database-migrations-beta) with `DISABLE_SCHEMA_UPDATE=true` on your pods, migrations will **not** auto-run on restart. You will need to handle migration cleanup manually (see Step 5) or re-run the PreSync hook against the older chart version.
## 4. Revert Application Version
Revert your deployment to the previous stable Docker image or Helm chart version.
### Docker
Update your deployment manifest (e.g., K8s Deployment, Docker Compose) to use the previous version:
```yaml
# Example: Reverting to the previous stable release
image: docker.litellm.ai/berriai/litellm:main-v<VERSION>-stable
```
See [all available images](https://github.com/orgs/BerriAI/packages).
### Helm
If you deployed via Helm, use `helm rollback`:
```bash
helm rollback <release-name> [revision-number]
```
## 5. Handle Database Migrations
If you are rolling back to a version that did not have a specific migration, you may need to resolve the migration state in the database.
> LiteLLM uses `prisma migrate deploy` for production (enabled via `USE_PRISMA_MIGRATE=True`). If a migration partially failed or you are reverting code that expects an older schema, you need to clean up the migration history in the `_prisma_migrations` table. See [Best Practices for Production](../proxy/prod#9-use-prisma-migrate-deploy).
### Option A — Delete stale migration entries (recommended)
Connect to your PostgreSQL database and remove migration entries that belong to the version you are rolling back from. This lets LiteLLM re-apply them cleanly if you upgrade again later.
```sql
-- View recent migrations
SELECT migration_name, finished_at, rolled_back_at, logs
FROM "_prisma_migrations"
ORDER BY started_at DESC
LIMIT 10;
-- Delete migration entries from the version you are rolling back from
DELETE FROM "_prisma_migrations"
WHERE migration_name = '<migration_name_from_newer_version>';
```
After deleting the entries, restart LiteLLM — it will re-apply the correct migrations for its version on startup.
> **Note:** If you have `DISABLE_SCHEMA_UPDATE=true` set on your pods, migrations will not auto-run. You need to either temporarily set it to `false`, or re-run the Helm PreSync migration job targeting the older version.
### Option B — Use `prisma migrate resolve` (if you have CLI access)
If you have access to the Prisma CLI (e.g., in a local development environment or a debug container with the `litellm-proxy-extras` package installed):
```bash
DATABASE_URL="<your_database_url>" prisma migrate resolve --rolled-back "<migration_name>"
```
> **Note:** This requires the Prisma CLI to be available in your environment (installed via `prisma-client-py`). If you don't have CLI access (e.g., no shell into the running container), use **Option A** (direct SQL) instead.
### Auto-Recovery Logic
LiteLLM's internal `ProxyExtrasDBManager` automatically attempts to handle idempotent migrations. In many cases, simply rolling back the version and restarting the proxy will be enough if the database changes are additive (e.g., new columns or tables).
## 6. Verification Checklist
After rolling back, verify the health of the system:
- [ ] **Health Endpoint**: Confirm the `/health` endpoint returns `200 OK`.
- [ ] **Check Logs**: Ensure no Prisma errors appear — look for `relation "..." does not exist`, `column "..." does not exist`, or `prisma migrate` failures in the logs.
- [ ] **Spend Tracking**: Run a test completion and confirm the spend is recorded in the `LiteLLM_SpendLogs` table.
- [ ] **Billing (Lago)**: If using Lago for billing (e.g., Lago → Stripe), check proxy logs for `Logged Lago Object` to confirm usage events are being sent.
- [ ] **State Consistency**: If using Redis for caching or rate limiting, consider clearing the cache if the newer version changed the cache key structure.
- [ ] **Admin UI**: Verify the Admin UI loads and shows correct data for keys and teams.
## 7. Troubleshooting
### "New migrations cannot be applied"
If you see this error after a rollback, it means the database has a migration in a "failed" state.
1. Identify the failed migration name (see the SQL query in Step 5).
2. Delete the failed entry from `_prisma_migrations`.
3. Restart the proxy.
### "relation X does not exist"
This typically means a migration entry exists in `_prisma_migrations` but the actual table/column was never created or was dropped.
1. Delete the stale migration entry.
2. Restart LiteLLM so it re-runs the migration.
For more details on Prisma errors, see [Prisma Migrations Troubleshoot](prisma_migrations).

2
docs/my-website/docs/tutorials/compare_llms.md
View file

@ -82,7 +82,7 @@ Benchmark Results for 'When will BerriAI IPO?':
+-----------------+----------------------------------------------------------------------------------+---------------------------+------------+
```
## Support
**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
<!--

406
docs/my-website/docs/tutorials/google_genai_sdk.md Normal file
View file

@ -0,0 +1,406 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Google GenAI SDK with LiteLLM
Use Google's official GenAI SDK (JavaScript/TypeScript and Python) with any LLM provider through LiteLLM Proxy.
The Google GenAI SDK (`@google/genai` for JS, `google-genai` for Python) provides a native interface for calling Gemini models. By pointing it to LiteLLM, you can use the same SDK with OpenAI, Anthropic, Bedrock, Azure, Vertex AI, or any other provider — while keeping the native Gemini request/response format.
## Why Use LiteLLM with Google GenAI SDK?
**Developer Benefits:**
- **Universal Model Access**: Use any LiteLLM-supported model (Anthropic, OpenAI, Vertex AI, Bedrock, etc.) through the Google GenAI SDK interface
- **Higher Rate Limits & Reliability**: Load balance across multiple models and providers to avoid hitting individual provider limits, with fallbacks to ensure you get responses even if one provider fails
**Proxy Admin Benefits:**
- **Centralized Management**: Control access to all models through a single LiteLLM proxy instance without giving developers API keys to each provider
- **Budget Controls**: Set spending limits and track costs across all SDK usage
- **Logging & Observability**: Track all requests with cost tracking, logging, and analytics
| Feature | Supported | Notes |
|---------|-----------|-------|
| Cost Tracking | ✅ | All models on `/generateContent` endpoint |
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | `streamGenerateContent` supported |
| Virtual Keys | ✅ | Use LiteLLM keys instead of Google keys |
| Load Balancing | ✅ | Via native router endpoints |
| Fallbacks | ✅ | Via native router endpoints |
## Quick Start
### 1. Install the SDK
<Tabs>
<TabItem value="js" label="JavaScript/TypeScript">
```bash
npm install @google/genai
```
</TabItem>
<TabItem value="python" label="Python">
```bash
pip install google-genai
```
</TabItem>
</Tabs>
### 2. Start LiteLLM Proxy
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: gemini-2.5-flash
litellm_params:
model: gemini/gemini-2.5-flash
api_key: os.environ/GEMINI_API_KEY
```
```bash
litellm --config config.yaml
```
### 3. Call the SDK through LiteLLM
<Tabs>
<TabItem value="js" label="JavaScript/TypeScript">
```javascript title="index.js" showLineNumbers
const { GoogleGenAI } = require("@google/genai");
const ai = new GoogleGenAI({
apiKey: "sk-1234", // LiteLLM virtual key (not a Google key)
httpOptions: {
baseUrl: "http://localhost:4000/gemini", // LiteLLM proxy URL
},
});
async function main() {
const response = await ai.models.generateContent({
model: "gemini-2.5-flash",
contents: "Explain how AI works",
});
console.log(response.text);
}
main();
```
</TabItem>
<TabItem value="python" label="Python">
```python title="main.py" showLineNumbers
from google import genai
client = genai.Client(
api_key="sk-1234", # LiteLLM virtual key (not a Google key)
http_options={"base_url": "http://localhost:4000/gemini"}, # LiteLLM proxy URL
)
response = client.models.generate_content(
model="gemini-2.5-flash",
contents="Explain how AI works",
)
print(response.text)
```
</TabItem>
<TabItem value="curl" label="curl">
```bash
curl "http://localhost:4000/gemini/v1beta/models/gemini-2.5-flash:generateContent?key=sk-1234" \
-H 'Content-Type: application/json' \
-X POST \
-d '{
"contents": [{
"parts": [{"text": "Explain how AI works"}]
}]
}'
```
</TabItem>
</Tabs>
## Streaming
<Tabs>
<TabItem value="js" label="JavaScript/TypeScript">
```javascript title="streaming.js" showLineNumbers
const { GoogleGenAI } = require("@google/genai");
const ai = new GoogleGenAI({
apiKey: "sk-1234",
httpOptions: {
baseUrl: "http://localhost:4000/gemini",
},
});
async function main() {
const response = await ai.models.generateContentStream({
model: "gemini-2.5-flash",
contents: "Write a short poem about the ocean",
});
for await (const chunk of response) {
process.stdout.write(chunk.text);
}
}
main();
```
</TabItem>
<TabItem value="python" label="Python">
```python title="streaming.py" showLineNumbers
from google import genai
client = genai.Client(
api_key="sk-1234",
http_options={"base_url": "http://localhost:4000/gemini"},
)
response = client.models.generate_content_stream(
model="gemini-2.5-flash",
contents="Write a short poem about the ocean",
)
for chunk in response:
print(chunk.text, end="")
```
</TabItem>
</Tabs>
## Multi-turn Chat
<Tabs>
<TabItem value="js" label="JavaScript/TypeScript">
```javascript title="chat.js" showLineNumbers
const { GoogleGenAI } = require("@google/genai");
const ai = new GoogleGenAI({
apiKey: "sk-1234",
httpOptions: {
baseUrl: "http://localhost:4000/gemini",
},
});
async function main() {
const chat = ai.chats.create({
model: "gemini-2.5-flash",
});
const response1 = await chat.sendMessage({ message: "I have 2 dogs and 3 cats." });
console.log(response1.text);
const response2 = await chat.sendMessage({ message: "How many pets is that in total?" });
console.log(response2.text);
}
main();
```
</TabItem>
<TabItem value="python" label="Python">
```python title="chat.py" showLineNumbers
from google import genai
client = genai.Client(
api_key="sk-1234",
http_options={"base_url": "http://localhost:4000/gemini"},
)
chat = client.chats.create(model="gemini-2.5-flash")
response1 = chat.send_message("I have 2 dogs and 3 cats.")
print(response1.text)
response2 = chat.send_message("How many pets is that in total?")
print(response2.text)
```
</TabItem>
</Tabs>
## Advanced: Use Any Model with the GenAI SDK
By default, the GenAI SDK talks to Gemini models. But with LiteLLM's router, you can route GenAI SDK requests to **any provider** — Anthropic, OpenAI, Bedrock, etc.
This works by using `model_group_alias` to map Gemini model names to your desired provider models. LiteLLM handles the format translation internally.
:::info
For this to work, point the SDK `baseUrl` to `http://localhost:4000` (without `/gemini`). This routes requests through LiteLLM's native Google endpoints, which go through the router and support model aliasing.
:::
<Tabs>
<TabItem value="anthropic" label="Anthropic">
Route `gemini-2.5-flash` requests to Claude Sonnet:
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: os.environ/ANTHROPIC_API_KEY
router_settings:
model_group_alias: {"gemini-2.5-flash": "claude-sonnet"}
```
</TabItem>
<TabItem value="openai" label="OpenAI">
Route `gemini-2.5-flash` requests to GPT-4o:
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: gpt-4o-model
litellm_params:
model: gpt-4o
api_key: os.environ/OPENAI_API_KEY
router_settings:
model_group_alias: {"gemini-2.5-flash": "gpt-4o-model"}
```
</TabItem>
<TabItem value="bedrock" label="Bedrock">
Route `gemini-2.5-flash` requests to Claude on Bedrock:
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
router_settings:
model_group_alias: {"gemini-2.5-flash": "bedrock-claude"}
```
</TabItem>
<TabItem value="multi" label="Multi-Provider Load Balancing">
Load balance across Anthropic and OpenAI:
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: my-model
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: my-model
litellm_params:
model: gpt-4o
api_key: os.environ/OPENAI_API_KEY
router_settings:
model_group_alias: {"gemini-2.5-flash": "my-model"}
```
</TabItem>
</Tabs>
Then use the SDK with `baseUrl` pointing to LiteLLM (without `/gemini`):
<Tabs>
<TabItem value="js" label="JavaScript/TypeScript">
```javascript title="any_model.js" showLineNumbers
const { GoogleGenAI } = require("@google/genai");
const ai = new GoogleGenAI({
apiKey: "sk-1234",
httpOptions: {
baseUrl: "http://localhost:4000", // No /gemini — goes through the router
},
});
async function main() {
// This calls Claude/GPT-4o/Bedrock under the hood via model_group_alias
const response = await ai.models.generateContent({
model: "gemini-2.5-flash",
contents: "Hello from any model!",
});
console.log(response.text);
}
main();
```
</TabItem>
<TabItem value="python" label="Python">
```python title="any_model.py" showLineNumbers
from google import genai
client = genai.Client(
api_key="sk-1234",
http_options={"base_url": "http://localhost:4000"}, # No /gemini
)
# This calls Claude/GPT-4o/Bedrock under the hood via model_group_alias
response = client.models.generate_content(
model="gemini-2.5-flash",
contents="Hello from any model!",
)
print(response.text)
```
</TabItem>
</Tabs>
## Pass-through vs Native Router Endpoints
LiteLLM offers two ways to handle GenAI SDK requests:
| | Pass-through (`/gemini`) | Native Router (`/`) |
|---|---|---|
| **baseUrl** | `http://localhost:4000/gemini` | `http://localhost:4000` |
| **Models** | Gemini only | Any provider via `model_group_alias` |
| **Translation** | None — proxies directly to Google | Translates internally |
| **Cost Tracking** | ✅ | ✅ |
| **Virtual Keys** | ✅ | ✅ |
| **Load Balancing** | ❌ | ✅ |
| **Fallbacks** | ❌ | ✅ |
| **Best for** | Simple Gemini proxy | Multi-provider routing |
## Environment Variable Configuration
You can also configure the SDK via environment variables instead of code:
```bash
# For JavaScript SDK (@google/genai)
export GOOGLE_GEMINI_BASE_URL="http://localhost:4000/gemini"
export GEMINI_API_KEY="sk-1234"
# For Python SDK (google-genai)
# Note: The Python SDK does not support a base URL env var.
# Configure it in code with http_options={"base_url": "..."} instead.
export GEMINI_API_KEY="sk-1234"
```
This is especially useful for tools built on top of the GenAI SDK (like [Gemini CLI](./litellm_gemini_cli.md)).
## Related Resources
- [Gemini CLI with LiteLLM](./litellm_gemini_cli.md)
- [Google AI Studio Pass-Through](../pass_through/google_ai_studio)
- [Google ADK with LiteLLM](./google_adk.md)
- [LiteLLM Proxy Quick Start](../proxy/quick_start)
- [`@google/genai` npm package](https://www.npmjs.com/package/@google/genai)
- [`google-genai` PyPI package](https://pypi.org/project/google-genai/)

373
docs/my-website/docs/tutorials/openai_agents_sdk.md Normal file
View file

@ -0,0 +1,373 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# OpenAI Agents SDK with LiteLLM
Use OpenAI's Agents SDK with any LLM provider through LiteLLM Proxy.
This tutorial shows you how to build AI agents using the OpenAI Agents SDK with support for multiple LLM providers through LiteLLM.
## Overview
The OpenAI Agents SDK provides a high-level interface for building AI agents. By integrating with LiteLLM, you can:
- Use multiple LLM providers (Bedrock, Azure, Vertex AI, etc.) with the same agent code
- Switch easily between models from different providers
- Connect to a LiteLLM proxy for centralized model management
:::tip Built-in LiteLLM Extension
The OpenAI Agents SDK includes an official LiteLLM extension (`LitellmModel`) that works without a proxy. If you don't need centralized proxy features (cost tracking, rate limiting, load balancing), you can use it directly:
```python
from agents import Agent, Runner
from agents.extensions.models.litellm_model import LitellmModel
agent = Agent(
name="Assistant",
instructions="You are a helpful assistant.",
model=LitellmModel(model="anthropic/claude-sonnet-4-20250514"),
)
result = Runner.run_sync(agent, "Hello!")
print(result.final_output)
```
See the [Docs](https://openai.github.io/openai-agents-python/models/litellm/) for more details. The rest of this tutorial focuses on the **proxy-based approach** for teams that need centralized model management.
:::
## Prerequisites
- Python environment setup
- API keys for your LLM providers
- Basic understanding of LLMs and agent concepts
## Installation
```bash showLineNumbers title="Install dependencies"
pip install openai-agents litellm
```
## 1. Start LiteLLM Proxy
Configure and start the LiteLLM proxy with the models you want to use:
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: bedrock-claude-sonnet-4
litellm_params:
model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
aws_region_name: "us-east-1"
- model_name: gpt-4o
litellm_params:
model: "openai/gpt-4o"
- model_name: claude-sonnet-4
litellm_params:
model: "anthropic/claude-sonnet-4-20250514"
- model_name: bedrock-claude-haiku
litellm_params:
model: "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0"
aws_region_name: "us-east-1"
- model_name: bedrock-nova-premier
litellm_params:
model: "bedrock/amazon.nova-premier-v1:0"
aws_region_name: "us-east-1"
```
```bash
litellm --config config.yaml
```
Required environment variables:
| Variable | Value | Description |
|----------|-------|-------------|
| `LITELLM_BASE_URL` | `http://localhost:4000` | LiteLLM proxy URL |
| `LITELLM_API_KEY` | `sk-1234` | Your LiteLLM API key (not your provider's key) |
## 2. Setting Up Environment
Import the necessary libraries and configure your LiteLLM proxy connection:
```python showLineNumbers title="Setup environment"
from __future__ import annotations
import asyncio
import os
from openai import AsyncOpenAI
from agents import (
Agent,
Model,
ModelProvider,
OpenAIChatCompletionsModel,
RunConfig,
Runner,
function_tool,
set_tracing_disabled,
)
# Point to LiteLLM proxy
BASE_URL = os.getenv("LITELLM_BASE_URL") or "http://localhost:4000"
API_KEY = os.getenv("LITELLM_API_KEY") or "sk-1234"
# Define model constants for cleaner code
MODEL_BEDROCK_SONNET = "bedrock-claude-sonnet-4"
MODEL_BEDROCK_HAIKU = "bedrock-claude-haiku"
MODEL_GPT_4O = "gpt-4o"
# Create the OpenAI client pointed at LiteLLM
client = AsyncOpenAI(base_url=BASE_URL, api_key=API_KEY)
# Disable tracing since we're not using OpenAI's platform directly
set_tracing_disabled(disabled=True)
```
## 3. Create a Custom Model Provider
The Agents SDK uses a `ModelProvider` to resolve model names. Create a custom provider that routes all requests through LiteLLM:
```python showLineNumbers title="Custom LiteLLM model provider"
class LiteLLMModelProvider(ModelProvider):
def get_model(self, model_name: str | None) -> Model:
return OpenAIChatCompletionsModel(
model=model_name or MODEL_BEDROCK_SONNET,
openai_client=client,
)
LITELLM_MODEL_PROVIDER = LiteLLMModelProvider()
```
## 4. Define a Simple Tool
Create a tool that your agent can use:
```python showLineNumbers title="Weather tool implementation"
@function_tool
def get_weather(city: str) -> str:
"""Retrieves the current weather report for a specified city.
Args:
city: The name of the city (e.g., "New York", "London", "Tokyo").
Returns:
A string containing the weather information for the city.
"""
print(f"[debug] getting weather for {city}")
mock_weather_db = {
"new york": "The weather in New York is sunny with a temperature of 25°C.",
"london": "It's cloudy in London with a temperature of 15°C.",
"tokyo": "Tokyo is experiencing light rain and a temperature of 18°C.",
}
city_normalized = city.lower()
if city_normalized in mock_weather_db:
return mock_weather_db[city_normalized]
else:
return f"Sorry, I don't have weather information for '{city}'."
```
## 5. Using Different Models with Agents
### 5.1 Using Bedrock Models
```python showLineNumbers title="Bedrock model via LiteLLM proxy"
async def test_bedrock_agent():
print("\n--- Testing Bedrock Claude Agent ---")
agent = Agent(
name="weather_agent_bedrock",
instructions="You are a helpful weather assistant powered by Claude. "
"Use the 'get_weather' tool for city weather requests. "
"Present information clearly.",
tools=[get_weather],
)
result = await Runner.run(
agent,
"What's the weather in Tokyo?",
run_config=RunConfig(
model_provider=LITELLM_MODEL_PROVIDER,
model="bedrock-claude-sonnet-4", # Uses the model name from your LiteLLM config
),
)
print(f"<<< Agent Response: {result.final_output}")
asyncio.run(test_bedrock_agent())
```
### 5.2 Using OpenAI Models
```python showLineNumbers title="OpenAI model via LiteLLM proxy"
async def test_openai_agent():
print("\n--- Testing OpenAI GPT Agent ---")
agent = Agent(
name="weather_agent_gpt",
instructions="You are a helpful weather assistant powered by GPT-4o. "
"Use the 'get_weather' tool for city weather requests. "
"Present information clearly.",
tools=[get_weather],
)
result = await Runner.run(
agent,
"What's the weather in London?",
run_config=RunConfig(
model_provider=LITELLM_MODEL_PROVIDER,
model="gpt-4o", # Uses the model name from your LiteLLM config
),
)
print(f"<<< Agent Response: {result.final_output}")
asyncio.run(test_openai_agent())
```
### 5.3 Using Anthropic Models
```python showLineNumbers title="Anthropic model via LiteLLM proxy"
async def test_anthropic_agent():
print("\n--- Testing Anthropic Claude Agent ---")
agent = Agent(
name="weather_agent_claude",
instructions="You are a helpful weather assistant powered by Claude. "
"Use the 'get_weather' tool for city weather requests. "
"Present information clearly.",
tools=[get_weather],
)
result = await Runner.run(
agent,
"What's the weather in New York?",
run_config=RunConfig(
model_provider=LITELLM_MODEL_PROVIDER,
model="claude-sonnet-4", # Uses the model name from your LiteLLM config
),
)
print(f"<<< Agent Response: {result.final_output}")
asyncio.run(test_anthropic_agent())
```
## 6. Complete Working Example
Here's a full end-to-end script you can copy and run:
```python showLineNumbers title="complete_agent.py"
from __future__ import annotations
import asyncio
import os
from openai import AsyncOpenAI
from agents import (
Agent,
Model,
ModelProvider,
OpenAIChatCompletionsModel,
RunConfig,
Runner,
function_tool,
set_tracing_disabled,
)
# Point to LiteLLM proxy
BASE_URL = os.getenv("LITELLM_BASE_URL") or "http://localhost:4000"
API_KEY = os.getenv("LITELLM_API_KEY") or "sk-1234"
MODEL_NAME = os.getenv("MODEL_NAME") or "bedrock-claude-sonnet-4"
client = AsyncOpenAI(base_url=BASE_URL, api_key=API_KEY)
set_tracing_disabled(disabled=True)
class LiteLLMModelProvider(ModelProvider):
def get_model(self, model_name: str | None) -> Model:
return OpenAIChatCompletionsModel(
model=model_name or MODEL_NAME,
openai_client=client,
)
LITELLM_MODEL_PROVIDER = LiteLLMModelProvider()
@function_tool
def get_weather(city: str) -> str:
"""Retrieves the current weather report for a specified city."""
print(f"[debug] getting weather for {city}")
mock_weather_db = {
"new york": "The weather in New York is sunny with a temperature of 25°C.",
"london": "It's cloudy in London with a temperature of 15°C.",
"tokyo": "Tokyo is experiencing light rain and a temperature of 18°C.",
}
city_normalized = city.lower()
if city_normalized in mock_weather_db:
return mock_weather_db[city_normalized]
else:
return f"Sorry, I don't have weather information for '{city}'."
async def main():
agent = Agent(
name="Assistant",
instructions="You are a helpful weather assistant. "
"Use the 'get_weather' tool for city weather requests. "
"Present information clearly and concisely.",
tools=[get_weather],
)
# Run with the default model (bedrock-claude-sonnet-4)
result = await Runner.run(
agent,
"What's the weather in Tokyo?",
run_config=RunConfig(model_provider=LITELLM_MODEL_PROVIDER),
)
print(result.final_output)
# Switch to a different model by passing model in RunConfig
result = await Runner.run(
agent,
"What's the weather in London?",
run_config=RunConfig(
model_provider=LITELLM_MODEL_PROVIDER,
model="gpt-4o",
),
)
print(result.final_output)
if __name__ == "__main__":
asyncio.run(main())
```
## Why Use LiteLLM with Agents SDK?
| Feature | Benefit |
|---------|---------|
| **Multi-Provider** | Use the same agent code with OpenAI, Bedrock, Azure, Vertex AI, etc. |
| **Cost Tracking** | Track spending across all agent conversations |
| **Rate Limiting** | Set budgets and limits on agent usage |
| **Load Balancing** | Distribute requests across multiple API keys or regions |
| **Fallbacks** | Automatically retry with different models if one fails |
## Related Resources
- [OpenAI Agents SDK Documentation](https://openai.github.io/openai-agents-python/)
- [LiteLLM Proxy Quick Start](../proxy/quick_start)

210
docs/my-website/docs/tutorials/openclaw_integration.md Normal file
View file

@ -0,0 +1,210 @@
---
sidebar_label: "OpenClaw"
---
# OpenClaw + LiteLLM Integration
[OpenClaw](https://openclaw.ai) is a self-hosted AI assistant that connects chat apps (WhatsApp, Telegram, Discord, and more) to LLM providers. By routing OpenClaw through LiteLLM Proxy, you get access to 100+ providers, cost tracking, spend limits, and automatic failover — all from a single gateway.
## What you'll set up
```
Chat apps → OpenClaw Gateway → LiteLLM Proxy → LLM Providers (OpenAI, Anthropic, etc.)
```
## Prerequisites
| Requirement | How to get it |
|---|---|
| **Node.js 22+** | `node --version` — install from [nodejs.org](https://nodejs.org) if needed |
| **Python 3.8+** | `python --version` |
| **At least one LLM API key** | OpenAI, Anthropic, Gemini, etc. |
## Step 1 — Install LiteLLM Proxy
```bash
pip install 'litellm[proxy]'
```
## Step 2 — Create a LiteLLM config file
Create a config file `litellm_config.yaml` with the models you want to use. Here's an example with OpenAI:
```yaml title="litellm_config.yaml"
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
general_settings:
master_key: sk-your-secret-key # pick any value — this is YOUR proxy password
```
:::tip Multi-provider example
You can add as many models as you want from different providers:
```yaml title="litellm_config.yaml"
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: gemini-flash
litellm_params:
model: gemini/gemini-2.0-flash
api_key: os.environ/GEMINI_API_KEY
general_settings:
master_key: sk-your-secret-key
```
See [LiteLLM proxy config docs](https://docs.litellm.ai/docs/proxy/configs) for all options.
:::
## Step 3 — Start the proxy
Make sure your API key(s) are available as environment variables (via `export`, `.env` file, or however you manage secrets), then start the proxy:
```bash
litellm --config litellm_config.yaml --port 4000
```
## Step 4 — Install OpenClaw
```bash
# macOS / Linux
curl -fsSL https://openclaw.ai/install.sh | bash
```
:::note Windows
On Windows, use PowerShell: `iwr -useb https://openclaw.ai/install.ps1 | iex`
WSL2 is recommended over native Windows.
:::
## Step 5 — Connect OpenClaw to LiteLLM
Run the onboarding wizard:
```bash
openclaw onboard --install-daemon
```
When prompted:
1. Choose **QuickStart** or **Manual** as the onboarding mode (both work — Manual gives you more options for gateway settings)
2. Select **LiteLLM** as the model/auth provider
3. Enter your LiteLLM `master_key` from Step 2 and set the base URL to your proxy address (e.g., `http://localhost:4000`)
4. When asked for the default model, choose **Enter model manually** and type the model name from your `litellm_config.yaml` (e.g., `litellm/gpt-4o`)
You can also set or change the model after onboarding:
```bash
openclaw models set litellm/gpt-4o
```
For scripted / CI environments, you can skip the prompts entirely:
```bash
openclaw onboard --non-interactive --accept-risk \
--auth-choice litellm-api-key \
--litellm-api-key "sk-your-secret-key" \
--custom-base-url "http://localhost:4000" \
--install-daemon --skip-channels --skip-skills
```
## Step 6 — Verify
Check the gateway is healthy:
```bash
openclaw health
```
Then send a test message:
```bash
openclaw dashboard # web UI
openclaw tui # terminal UI
openclaw agent --agent main -m "Hello, what model are you?" # one-shot CLI
```
If you get a response from your model, the integration is working.
Check which model is active:
```bash
openclaw models status
```
## Config reference
After onboarding, OpenClaw stores the LiteLLM provider config in `~/.openclaw/openclaw.json`. The relevant sections are something like this:
```json5 title="~/.openclaw/openclaw.json (excerpt)"
{
"models": {
"providers": {
"litellm": {
"baseUrl": "http://localhost:4000",
"apiKey": "sk-your-secret-key",
"api": "openai-completions",
"models": [
{
"id": "gpt-4o",
"name": "GPT-4o via LiteLLM"
}
]
}
}
},
"agents": {
"defaults": {
"model": { "primary": "litellm/gpt-4o" }
}
}
}
```
You can edit this file directly to add more models or change the `baseUrl`. OpenClaw hot-reloads changes automatically.
## Troubleshooting
**Connection refused / proxy not reachable**
Make sure the LiteLLM proxy is running and that the `baseUrl` in your OpenClaw config matches:
```bash
curl http://localhost:4000/health -H "Authorization: Bearer sk-your-secret-key"
```
**Wrong model or "Invalid model name"**
The model name in OpenClaw must match a `model_name` from your `litellm_config.yaml`. Switch the active model with:
```bash
openclaw models set litellm/gpt-4o
```
**Gateway pairing issues after reinstall**
If the CLI can't connect to the gateway after a reinstall, stop the service and reinstall it:
```bash
openclaw gateway stop
openclaw gateway install
```
## References
- [OpenClaw docs](https://docs.openclaw.ai)
- [OpenClaw LiteLLM provider docs](https://docs.openclaw.ai/providers/litellm)
- [OpenClaw model providers](https://docs.openclaw.ai/concepts/model-providers)
- [LiteLLM proxy configuration](https://docs.litellm.ai/docs/proxy/configs)

40
docs/my-website/docs/tutorials/presidio_pii_masking.md
View file

@ -592,6 +592,21 @@ def test_pii_masking_allows_normal_text():
## Part 7: Troubleshooting
### Issue: Guardrail failure: non-JSON response from Presidio
**Symptom:** You receive an error indicating `expected application/json Content-Type but received text/html` or similar.
**Root cause:** Your ingress controller or reverse proxy might be routing the `/analyze` or `/anonymize` POST request to a health endpoint (like `/health` or `/presidio-analyzer/health`) which returns plain text instead of JSON.
**Fix:** Ensure your `PRESIDIO_ANALYZER_API_BASE` and `PRESIDIO_ANONYMIZER_API_BASE` are correctly pointing directly to the Presidio API endpoints, or that your ingress routes the path correctly without stripping it and inadvertently forwarding to a plain-text health check endpoint.
**Verification:** You can verify your endpoints using `curl`. It should return a JSON array, not `text/html`:
```bash
curl -sv -X POST http://your-analyzer-endpoint/analyze \
-H "Content-Type: application/json" \
-d '{"text":"test","language":"en"}'
```
### Issue: Presidio Not Detecting PII
**Check 1: Language Configuration**
@ -685,3 +700,28 @@ Congratulations! 🎉 You've successfully set up PII masking with Presidio and L
---
**Need help?** Join our [Discord community](https://discord.com/invite/wuPM9dRgDw) or open an issue on GitHub!
### Suppressing False Positives
Presidio can sometimes trigger false positive detections. For example, short alphanumeric strings might be incorrectly flagged as `US_DRIVER_LICENSE`.
You can suppress these false positives using `presidio_score_thresholds` or `presidio_entities_deny_list`.
```yaml
guardrails:
- guardrail_name: presidio-pii
litellm_params:
guardrail: presidio
mode: "pre_call"
presidio_analyzer_api_base: "http://localhost:5002/"
presidio_anonymizer_api_base: "http://localhost:5001/"
# Use high score thresholds to reduce false positives
presidio_score_thresholds:
US_DRIVER_LICENSE: 0.85
ALL: 0.5
# Or exclude certain entity types entirely from detection
presidio_entities_deny_list:
- US_DRIVER_LICENSE
```

BIN
docs/my-website/img/litellm_proxy_setup.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 538 KiB

BIN
docs/my-website/img/release_notes/compliance_playground.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 780 KiB

BIN
docs/my-website/img/release_notes/guardrail_garden.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.2 MiB

Some files were not shown because too many files have changed in this diff Show more