scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-21 00:21:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 73

Merge branch 'BerriAI:main' into main

Browse source
This commit is contained in:
fzowl 2025-11-14 11:53:12 +01:00 • committed by GitHub
commit 958905a750
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1474 changed files with 138202 additions and 14412 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

294
.circleci/config.yml
View file

@ -1,8 +1,8 @@
version: 2.1
orbs:
codecov: codecov/codecov@4.0.1
node: circleci/node@5.1.0 # Add this line to declare the node orb
win: circleci/windows@5.0 # Add Windows orb
node: circleci/node@5.1.0 # Add this line to declare the node orb
win: circleci/windows@5.0 # Add Windows orb
commands:
setup_google_dns:
@ -50,7 +50,7 @@ jobs:
name: Run Windows-specific test
command: |
python -m pytest tests/windows_tests/test_litellm_on_windows.py -v
mypy_linting:
docker:
- image: cimg/python:3.12
@ -500,7 +500,7 @@ jobs:
paths:
- litellm_router_coverage.xml
- litellm_router_coverage
litellm_router_unit_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
@ -532,7 +532,7 @@ jobs:
command: |
pwd
ls
python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -563,8 +563,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.13
command: |
@ -1041,6 +1042,92 @@ jobs:
paths:
- llm_responses_api_coverage.xml
- llm_responses_api_coverage
ocr_testing:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml ocr_coverage.xml
mv .coverage ocr_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- ocr_coverage.xml
- ocr_coverage
search_testing:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml search_coverage.xml
mv .coverage search_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- search_coverage.xml
- search_coverage
litellm_mapped_tests:
docker:
- image: cimg/python:3.11
@ -1078,7 +1165,7 @@ jobs:
command: |
pwd
ls
python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -s -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8
python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -1310,7 +1397,7 @@ jobs:
command: |
pwd
ls
python -m pytest -vv tests/image_gen_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
python -m pytest -vv tests/image_gen_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -1376,6 +1463,49 @@ jobs:
paths:
- logging_coverage.xml
- logging_coverage
audio_testing:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml audio_coverage.xml
mv .coverage audio_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- audio_coverage.xml
- audio_coverage
installing_litellm_on_python:
docker:
- image: circleci/python:3.8
@ -1442,7 +1572,7 @@ jobs:
python -m pytest -vv tests/local_testing/test_basic_python_version.py
helm_chart_testing:
machine:
image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker
image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker
resource_class: medium
working_directory: ~/project
@ -1454,7 +1584,7 @@ jobs:
name: Install Helm
command: |
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
# Install kind
- run:
name: Install Kind
@ -1462,7 +1592,7 @@ jobs:
curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
chmod +x ./kind
sudo mv ./kind /usr/local/bin/kind
# Install kubectl
- run:
name: Install kubectl
@ -1470,19 +1600,19 @@ jobs:
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
chmod +x kubectl
sudo mv kubectl /usr/local/bin/
# Create kind cluster
- run:
name: Create Kind Cluster
command: |
kind create cluster --name litellm-test
# Run helm lint
- run:
name: Run helm lint
command: |
helm lint ./deploy/charts/litellm-helm
# Run helm tests
- run:
name: Run helm tests
@ -1491,22 +1621,21 @@ jobs:
# Wait for pod to be ready
echo "Waiting 30 seconds for pod to be ready..."
sleep 30
# Print pod logs before running tests
echo "Printing pod logs..."
kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
# Run the helm tests
helm test litellm --logs
helm test litellm --logs
# Cleanup
- run:
name: Cleanup
command: |
kind delete cluster --name litellm-test
when: always # This ensures cleanup runs even if previous steps fail
when: always # This ensures cleanup runs even if previous steps fail
check_code_and_doc_quality:
docker:
@ -1580,6 +1709,7 @@ jobs:
pip install "pytest==7.3.1"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install apscheduler
- run:
name: Build Docker image
command: |
@ -1617,7 +1747,7 @@ jobs:
echo "=== Printing Full Container Startup Logs ==="
docker logs my-app
echo "=== End of Full Container Startup Logs ==="
if docker logs my-app 2>&1 | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
echo "Expected message found in logs. Test passed."
else
@ -1630,7 +1760,6 @@ jobs:
python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
no_output_timeout: 120m
build_and_test:
machine:
image: ubuntu-2204:2023.10.1
@ -1642,8 +1771,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@ -1780,8 +1910,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@ -1922,8 +2053,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@ -2106,8 +2238,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@ -2214,8 +2347,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@ -2347,8 +2481,10 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
sudo systemctl restart docker
- run:
name: Install Python 3.9
command: |
@ -2434,8 +2570,7 @@ jobs:
pwd
ls
python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
no_output_timeout:
120m
no_output_timeout: 120m
- run:
name: Stop and remove containers
command: |
@ -2446,7 +2581,7 @@ jobs:
when: always
- store_test_results:
path: test-results
proxy_build_from_pip_tests:
# Change from docker to machine executor
machine:
@ -2480,6 +2615,7 @@ jobs:
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install "mypy==1.18.2"
pip install apscheduler
- run:
name: Build Docker image
command: |
@ -2555,8 +2691,9 @@ jobs:
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@ -2633,8 +2770,8 @@ jobs:
-e GEMINI_API_KEY=$GEMINI_API_KEY \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \
-e AZURE_API_KEY_PASSHROUGH=$AZURE_API_KEY_PASSHROUGH \
-e AZURE_API_BASE_PASSHROUGH=$AZURE_API_BASE_PASSHROUGH \
-e AZURE_API_KEY=$AZURE_API_KEY \
-e AZURE_API_BASE=$AZURE_API_BASE \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
@ -2663,17 +2800,17 @@ jobs:
curl -sSL https://rvm.io/mpapis.asc | gpg --import -
curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -
}
# Install Ruby version manager (RVM)
curl -sSL https://get.rvm.io | bash -s stable
# Source RVM from the correct location
source $HOME/.rvm/scripts/rvm
# Install Ruby 3.2.2
rvm install 3.2.2
rvm use 3.2.2 --default
# Install latest Bundler
gem install bundler
@ -2741,7 +2878,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
coverage combine llm_translation_coverage llm_responses_api_coverage mcp_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@ -2827,32 +2964,32 @@ jobs:
python -m pip install toml
# Get current version from pyproject.toml
CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
# Get last published version from PyPI
LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])")
echo "Current version: $CURRENT_VERSION"
echo "Last published version: $LAST_VERSION"
# Compare versions using Python's packaging.version
VERSION_COMPARE=$(python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)")
echo "Version compare: $VERSION_COMPARE"
if [ "$VERSION_COMPARE" = "1" ]; then
echo "Error: Current version ($CURRENT_VERSION) is less than last published version ($LAST_VERSION)"
exit 1
fi
# If versions are equal or current is greater, check contents
pip download --no-deps litellm-proxy-extras==$LAST_VERSION -d /tmp
echo "Contents of /tmp directory:"
ls -la /tmp
# Find the downloaded file (could be .whl or .tar.gz)
DOWNLOADED_FILE=$(ls /tmp/litellm_proxy_extras-*)
echo "Downloaded file: $DOWNLOADED_FILE"
# Extract based on file extension
if [[ "$DOWNLOADED_FILE" == *.whl ]]; then
echo "Extracting wheel file..."
@ -2863,10 +3000,10 @@ jobs:
tar -xzf "$DOWNLOADED_FILE" -C /tmp
EXTRACTED_DIR="/tmp/litellm_proxy_extras-$LAST_VERSION"
fi
echo "Contents of extracted package:"
ls -R "$EXTRACTED_DIR"
# Compare contents
if ! diff -r "$EXTRACTED_DIR/litellm_proxy_extras" ./litellm_proxy_extras; then
if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then
@ -2932,23 +3069,24 @@ jobs:
export NVM_DIR="/opt/circleci/.nvm"
source "$NVM_DIR/nvm.sh"
source "$NVM_DIR/bash_completion"
# Install and use Node version
nvm install v20
nvm use v20
cd ui/litellm-dashboard
# Install dependencies first
npm install
# Now source the build script
source ./build_ui.sh
- run:
name: Install Docker CLI (In case it's not already installed)
name: Upgrade Docker to v24.x (API 1.44+)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
@ -2996,10 +3134,10 @@ jobs:
source "$NVM_DIR/nvm.sh"
nvm install 20
nvm use 20
cd ui/litellm-dashboard
npm ci || npm install
# CI run, with both LCOV (Codecov) and HTML (artifact you can click)
CI=true npm run test -- --run --coverage \
--coverage.provider=v8 \
@ -3007,7 +3145,6 @@ jobs:
--coverage.reporter=html \
--coverage.reportsDirectory=coverage/html
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@ -3289,6 +3426,18 @@ workflows:
only:
- main
- /litellm_.*/
- ocr_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- search_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_enterprise_tests:
filters:
branches:
@ -3331,6 +3480,12 @@ workflows:
only:
- main
- /litellm_.*/
- audio_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- upload-coverage:
requires:
- llm_translation_testing
@ -3338,6 +3493,8 @@ workflows:
- google_generate_content_endpoint_testing
- guardrails_testing
- llm_responses_api_testing
- ocr_testing
- search_testing
- litellm_mapped_tests
- litellm_mapped_enterprise_tests
- batches_testing
@ -3345,6 +3502,7 @@ workflows:
- pass_through_unit_testing
- image_gen_testing
- logging_testing
- audio_testing
- litellm_router_testing
- litellm_router_unit_testing
- caching_unit_tests
@ -3400,6 +3558,8 @@ workflows:
- mcp_testing
- google_generate_content_endpoint_testing
- llm_responses_api_testing
- ocr_testing
- search_testing
- litellm_mapped_tests
- litellm_mapped_enterprise_tests
- batches_testing
@ -3407,6 +3567,7 @@ workflows:
- pass_through_unit_testing
- image_gen_testing
- logging_testing
- audio_testing
- litellm_router_testing
- litellm_router_unit_testing
- caching_unit_tests
@ -3428,4 +3589,3 @@ workflows:
- check_code_and_doc_quality
- publish_proxy_extras
- guardrails_testing

3
.circleci/requirements.txt
View file

@ -15,4 +15,5 @@ fastapi-sso==0.16.0
uvloop==0.21.0
mcp==1.10.1 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
fastuuid==0.12.0
fastuuid==0.12.0
responses==0.25.7 # for proxy client tests

46
.dockerignore
View file

@ -4,9 +4,51 @@ cookbook
.github
tests
.git
.github
.circleci
.devcontainer
*.tgz
log.txt
docker/Dockerfile.*
# Claude Flow generated files (must be excluded from Docker build)
.claude/
.claude-flow/
.swarm/
.hive-mind/
memory/
coordination/
claude-flow
.mcp.json
hive-mind-prompt-*.txt
# Python virtual environments and version managers
.venv/
venv/
**/.venv/
**/venv/
.python-version
.pyenv/
__pycache__/
**/__pycache__/
*.pyc
.mypy_cache/
.pytest_cache/
.ruff_cache/
**/pyvenv.cfg
# Common project exclusions
.vscode
*.pyo
*.pyd
.Python
env/
.pytest_cache
.coverage
htmlcov/
dist/
build/
*.egg-info/
.DS_Store
node_modules/
*.log
.env
.env.local

1
.github/workflows/interpret_load_test.py vendored
View file

@ -88,6 +88,7 @@ def get_docker_run_command(release_version):
if __name__ == "__main__":
return
csv_file = "load_test_stats.csv" # Change this to the path of your CSV file
markdown_table = interpret_results(csv_file)

3
.github/workflows/test-litellm.yml vendored
View file

@ -33,6 +33,7 @@ jobs:
poetry run pip install "google-genai==1.22.0"
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.18"
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
@ -40,4 +41,4 @@ jobs:
cd ..
- name: Run tests
run: |
poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4
poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

1
.gitignore vendored
View file

@ -99,3 +99,4 @@ litellm/proxy/to_delete_loadtest_work/*
update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
litellm/proxy/_experimental/out/guardrails/index.html
scripts/test_vertex_ai_search.py

2
CONTRIBUTING.md
View file

@ -258,7 +258,7 @@ docker run \
If you need help:
- 💬 [Join our Discord](https://discord.gg/wuPM9dRgDw)
- 💬 [Join our Slack](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
- 💬 [Join our Slack](https://www.litellm.ai/support)
- 📧 Email us: ishaan@berri.ai / krrish@berri.ai
- 🐛 [Create an issue](https://github.com/BerriAI/litellm/issues/new)

4
Dockerfile
View file

@ -65,6 +65,10 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Remove test files and keys from dependencies
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh

2
Makefile
View file

@ -45,7 +45,7 @@ install-proxy-dev-ci:
install-test-deps: install-proxy-dev
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
cd enterprise && python -m pip install -e . && cd ..
cd enterprise && poetry run pip install -e . && cd ..
install-helm-unittest:
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"

154
README.md
View file

@ -37,6 +37,8 @@ LiteLLM manages:
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs) <br>
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
@ -132,11 +134,15 @@ print(response)
## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
LiteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
```python
from litellm import completion
messages = [{"content": "Hello, how are you?", "role": "user"}]
# gpt-4o
response = completion(model="openai/gpt-4o", messages=messages, stream=True)
for part in response:
print(part.choices[0].delta.content or "")
@ -301,52 +307,108 @@ curl 'http://0.0.0.0:4000/key/generate' \
}
```
## Supported Providers ([Docs](https://docs.litellm.ai/docs/providers))
## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
| Provider | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) | [Async Image Generation](https://docs.litellm.ai/docs/image_generation) |
|-------------------------------------------------------------------------------------|---------------------------------------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------|-------------------------------------------------------------------------------|-------------------------------------------------------------------------|
| [openai](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [Meta - Llama API](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ | ✅ | | |
| [azure](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [AI/ML API](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [aws - bedrock](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [google - vertex_ai](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [google - palm](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | ✅ | | |
| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | | |
| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ | | |
| [CompactifAI](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | ✅ | | |
| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ | | |
| [empower](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | ✅ |
| [huggingface](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [replicate](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | ✅ | | |
| [together_ai](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | ✅ | | |
| [openrouter](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | ✅ | | |
| [ai21](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | ✅ | | |
| [baseten](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | ✅ | | |
| [vllm](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | ✅ | | |
| [nlp_cloud](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | ✅ | | |
| [aleph alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | ✅ | | |
| [petals](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | ✅ | | |
| [ollama](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ | | |
| [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ | | |
| [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ | | |
| [Deepseek](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | ✅ | | |
| [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ | | |
| [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ | |
| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | |
| [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | ✅ | | |
| [Galadriel](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | ✅ | | |
| [GradientAI](https://docs.litellm.ai/docs/providers/gradient_ai) | ✅ | ✅ | | | | |
| [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | ✅ | | |
| [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | ✅ | | |
| [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [Heroku](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | | | | |
| [OVHCloud AI Endpoints](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | | | | |
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
|-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
| [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [Aleph Alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | | | | | | | |
| [Anthropic (`anthropic`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
| [Anthropic Text (`anthropic_text`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
| [Anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | | | | | | | |
| [AssemblyAI (`assemblyai`)](https://docs.litellm.ai/docs/pass_through/assembly_ai) | ✅ | ✅ | ✅ | | | ✅ | | | | |
| [Auto Router (`auto_router`)](https://docs.litellm.ai/docs/proxy/auto_routing) | ✅ | ✅ | ✅ | | | | | | | |
| [AWS - Bedrock (`bedrock`)](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [AWS - Sagemaker (`sagemaker`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Azure (`azure`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [Azure AI (`azure_ai`)](https://docs.litellm.ai/docs/providers/azure_ai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [Azure Text (`azure_text`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Baseten (`baseten`)](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | | | | | | | |
| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | |
| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | |
| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | |
| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | |
| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Cohere (`cohere`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [Cohere Chat (`cohere_chat`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | | | | | | | |
| [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ | | | | | | | |
| [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | | | | | | | |
| [Databricks (`databricks`)](https://docs.litellm.ai/docs/providers/databricks) | ✅ | ✅ | ✅ | | | | | | | |
| [DataRobot (`datarobot`)](https://docs.litellm.ai/docs/providers/datarobot) | ✅ | ✅ | ✅ | | | | | | | |
| [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ | | | ✅ | | | | |
| [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | | | | | | | |
| [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | | | | | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [FriendliAI (`friendliai`)](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | | | | | | | |
| [Galadriel (`galadriel`)](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | | | | | | | |
| [GitHub Copilot (`github_copilot`)](https://docs.litellm.ai/docs/providers/github_copilot) | ✅ | ✅ | ✅ | | | | | | | |
| [GitHub Models (`github`)](https://docs.litellm.ai/docs/providers/github) | ✅ | ✅ | ✅ | | | | | | | |
| [Google - PaLM](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | | | | | | | |
| [Google - Vertex AI (`vertex_ai`)](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [Google AI Studio - Gemini (`gemini`)](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | | | | | | | |
| [GradientAI (`gradient_ai`)](https://docs.litellm.ai/docs/providers/gradient_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Groq AI (`groq`)](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | | | | | | | |
| [Heroku (`heroku`)](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | ✅ | | | | | | | |
| [Hosted VLLM (`hosted_vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |
| [Huggingface (`huggingface`)](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [Hyperbolic (`hyperbolic`)](https://docs.litellm.ai/docs/providers/hyperbolic) | ✅ | ✅ | ✅ | | | | | | | |
| [IBM - Watsonx.ai (`watsonx`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Infinity (`infinity`)](https://docs.litellm.ai/docs/providers/infinity) | | | | ✅ | | | | | | |
| [Jina AI (`jina_ai`)](https://docs.litellm.ai/docs/providers/jina_ai) | | | | ✅ | | | | | | |
| [Lambda AI (`lambda_ai`)](https://docs.litellm.ai/docs/providers/lambda_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Lemonade (`lemonade`)](https://docs.litellm.ai/docs/providers/lemonade) | ✅ | ✅ | ✅ | | | | | | | |
| [LiteLLM Proxy (`litellm_proxy`)](https://docs.litellm.ai/docs/providers/litellm_proxy) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [Llamafile (`llamafile`)](https://docs.litellm.ai/docs/providers/llamafile) | ✅ | ✅ | ✅ | | | | | | | |
| [LM Studio (`lm_studio`)](https://docs.litellm.ai/docs/providers/lm_studio) | ✅ | ✅ | ✅ | | | | | | | |
| [Maritalk (`maritalk`)](https://docs.litellm.ai/docs/providers/maritalk) | ✅ | ✅ | ✅ | | | | | | | |
| [Meta - Llama API (`meta_llama`)](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ | | | | | | | |
| [Mistral AI API (`mistral`)](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | |
| [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | |
| [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | | | | | | | |
| [Nscale (`nscale`)](https://docs.litellm.ai/docs/providers/nscale) | ✅ | ✅ | ✅ | | | | | | | |
| [Nvidia NIM (`nvidia_nim`)](https://docs.litellm.ai/docs/providers/nvidia_nim) | ✅ | ✅ | ✅ | | | | | | | |
| [OCI (`oci`)](https://docs.litellm.ai/docs/providers/oci) | ✅ | ✅ | ✅ | | | | | | | |
| [Ollama (`ollama`)](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [Ollama Chat (`ollama_chat`)](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | | | | | | | |
| [Oobabooga (`oobabooga`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [OpenAI (`openai`)](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) | | | | ✅ | | | | | | |
| [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | | | | | | | |
| [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | | | | | | | |
| [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | | | | | | | |
| [Predibase (`predibase`)](https://docs.litellm.ai/docs/providers/predibase) | ✅ | ✅ | ✅ | | | | | | | |
| [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) | | | | | ✅ | | | | | |
| [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | | | | | | | |
| [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | |
| [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | |
| [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | |
| [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
| [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | | | | | | | |
| [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ | | | | | | | |
| [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ | | | | | | | |
| [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ | | | | | | | |
| [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
| [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |
| [Volcengine (`volcengine`)](https://docs.litellm.ai/docs/providers/volcano) | ✅ | ✅ | ✅ | | | | | | | |
| [Voyage AI (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | |
| [WandB Inference (`wandb`)](https://docs.litellm.ai/docs/providers/wandb_inference) | ✅ | ✅ | ✅ | | | | | | | |
| [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | |
| [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | |
| [Xinference (`xinference`)](https://docs.litellm.ai/docs/providers/xinference) | | | | ✅ | | | | | | |
[**Read the Docs**](https://docs.litellm.ai/docs/)

261
VERTEX_ENV_SETUP.md Normal file
View file

@ -0,0 +1,261 @@
# Vertex AI Environment Variables Setup Guide
## Overview
LiteLLM can load Vertex AI credentials from environment variables instead of storing them in config files. This is more secure and easier to manage for local development.
## Environment Variables
LiteLLM looks for these environment variables (in order of precedence):
### 1. **DEFAULT_VERTEXAI_PROJECT** (Required)
Your GCP project ID that has Vertex AI enabled.
```bash
export DEFAULT_VERTEXAI_PROJECT="my-gcp-project-id"
```
### 2. **DEFAULT_VERTEXAI_LOCATION** (Required)
The region/location for Vertex AI services.
```bash
export DEFAULT_VERTEXAI_LOCATION="global"
# or
export DEFAULT_VERTEXAI_LOCATION="us-central1"
```
Common locations:
- `global` - For Discovery Engine and global services
- `us-central1` - US Central region
- `us-east1` - US East region
- `europe-west1` - Europe West region
- `asia-southeast1` - Asia Southeast region
### 3. **DEFAULT_GOOGLE_APPLICATION_CREDENTIALS** (Required)
Path to your service account JSON key file.
```bash
export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json"
```
### 4. **GOOGLE_APPLICATION_CREDENTIALS** (Fallback)
Standard Google Cloud environment variable (used as fallback).
```bash
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json"
```
## Quick Setup
### Option 1: Interactive Script
```bash
chmod +x setup_vertex_env.sh
source setup_vertex_env.sh
```
### Option 2: Manual Setup
1. **Set environment variables** (for current session):
```bash
export DEFAULT_VERTEXAI_PROJECT="your-project-id"
export DEFAULT_VERTEXAI_LOCATION="global"
export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"
export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"
```
2. **Make them persistent** (add to `~/.zshrc` or `~/.bashrc`):
```bash
echo 'export DEFAULT_VERTEXAI_PROJECT="your-project-id"' >> ~/.zshrc
echo 'export DEFAULT_VERTEXAI_LOCATION="global"' >> ~/.zshrc
echo 'export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"' >> ~/.zshrc
echo 'export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"' >> ~/.zshrc
```
3. **Reload your shell**:
```bash
source ~/.zshrc
```
## Service Account Setup
### 1. Create a Service Account
```bash
gcloud iam service-accounts create litellm-vertex-sa \
--display-name="LiteLLM Vertex AI Service Account"
```
### 2. Grant Necessary Permissions
For Discovery Engine (vector stores):
```bash
gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \
--member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \
--role="roles/discoveryengine.viewer"
gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \
--member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \
--role="roles/discoveryengine.dataStoreEditor"
```
For general Vertex AI:
```bash
gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \
--member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \
--role="roles/aiplatform.user"
```
### 3. Create and Download Key
```bash
gcloud iam service-accounts keys create ~/service-account-key.json \
--iam-account=litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com
```
## Verify Setup
### Check Environment Variables
```bash
python3 << 'EOF'
import os
print("✓ Environment Variables:")
print(f" DEFAULT_VERTEXAI_PROJECT: {os.getenv('DEFAULT_VERTEXAI_PROJECT')}")
print(f" DEFAULT_VERTEXAI_LOCATION: {os.getenv('DEFAULT_VERTEXAI_LOCATION')}")
print(f" DEFAULT_GOOGLE_APPLICATION_CREDENTIALS: {os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS')}")
print(f" GOOGLE_APPLICATION_CREDENTIALS: {os.getenv('GOOGLE_APPLICATION_CREDENTIALS')}")
# Check if credentials file exists
creds_path = os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS')
if creds_path and os.path.exists(creds_path):
print(f"\n✅ Credentials file found at: {creds_path}")
else:
print(f"\n❌ Credentials file NOT found at: {creds_path}")
EOF
```
### Test Authentication
```bash
python3 << 'EOF'
import os
import json
from google.oauth2 import service_account
from google.auth.transport.requests import Request
creds_path = os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS')
project = os.getenv('DEFAULT_VERTEXAI_PROJECT')
try:
# Load credentials
credentials = service_account.Credentials.from_service_account_file(
creds_path,
scopes=['https://www.googleapis.com/auth/cloud-platform']
)
# Get access token
credentials.refresh(Request())
print("✅ Authentication successful!")
print(f" Project: {project}")
print(f" Service Account: {credentials.service_account_email}")
print(f" Token expiry: {credentials.expiry}")
except Exception as e:
print(f"❌ Authentication failed: {e}")
EOF
```
## Using with Vector Store Passthrough
Once your environment is set up, the vector store passthrough will work in two ways:
### 1. **With Vector Store Config** (Priority 1)
If you have a vector store configured with its own credentials in `litellm_params`, those will be used first:
```yaml
vector_stores:
- vector_store_id: test-store-123
custom_llm_provider: vertex_ai
litellm_params:
vertex_project: "specific-project"
vertex_location: "us-central1"
vertex_credentials: "{...}" # Inline credentials
```
### 2. **Environment Variables Fallback** (Priority 2)
If the vector store doesn't have explicit credentials, it falls back to your environment variables:
```yaml
vector_stores:
- vector_store_id: test-store-123
custom_llm_provider: vertex_ai
# No litellm_params - will use DEFAULT_VERTEXAI_PROJECT, DEFAULT_VERTEXAI_LOCATION, etc.
```
### 3. **Model Config Fallback** (Priority 3)
If neither above work, it looks for credentials in your model configuration.
## Troubleshooting
### "No credentials found"
Check that all environment variables are set:
```bash
env | grep -E "(DEFAULT_VERTEXAI|GOOGLE_APPLICATION_CREDENTIALS)"
```
### "Authentication failed"
Verify your service account key is valid:
```bash
cat $DEFAULT_GOOGLE_APPLICATION_CREDENTIALS | python3 -m json.tool
```
### "Permission denied"
Ensure your service account has the necessary roles:
```bash
gcloud projects get-iam-policy YOUR_PROJECT_ID \
--flatten="bindings[].members" \
--filter="bindings.members:serviceAccount:litellm-vertex-sa@*"
```
### Different Credentials for Different Projects
If you need to use different credentials for different vector stores, configure them explicitly in the vector store config rather than relying on environment variables.
## Start LiteLLM Proxy
Once your environment is configured:
```bash
# Start the proxy (it will automatically load env vars)
litellm --config proxy_server_config.yaml
# Or with debug logging
export LITELLM_LOG=DEBUG
litellm --config proxy_server_config.yaml
```
You should see logs like:
```
Vertex: Loading vertex credentials from /path/to/service-account.json
Found credentials for vertex_ai_default
```
## Test the Endpoint
```bash
curl -X POST http://0.0.0.0:4000/vertex_ai/discovery/v1/projects/fake-project/locations/global/dataStores/test-store-123/servingConfigs/default_config:search \
-H 'Authorization: Bearer YOUR_LITELLM_API_KEY' \
-H 'Content-Type: application/json' \
-d '{"query": "test query"}'
```
The proxy will use your environment credentials to make the request to Vertex AI!

4
batch_small.jsonl Normal file
View file

@ -0,0 +1,4 @@
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello, how are you?"}]}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "What is the weather today?"}]}}
{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Tell me a short joke"}]}}

474
cookbook/LiteLLM_CometAPI.ipynb vendored Normal file
View file

File diff suppressed because one or more lines are too long

20
cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml Normal file
View file

@ -0,0 +1,20 @@
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
key_management_system: "custom"
key_management_settings:
custom_secret_manager: my_secret_manager.InMemorySecretManager
store_virtual_keys: true
prefix_for_stored_virtual_keys: "litellm/"
access_mode: "read_and_write"
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY # Read from custom secret manager
- model_name: claude-3-5-sonnet
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY # Read from custom secret manager

79
cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py Normal file
View file

@ -0,0 +1,79 @@
"""
Example custom secret manager for LiteLLM Proxy.
This is a simple in-memory secret manager for testing purposes.
In production, replace this with your actual secret management system.
"""
from typing import Optional, Union
import httpx
from litellm.integrations.custom_secret_manager import CustomSecretManager
class InMemorySecretManager(CustomSecretManager):
def __init__(self):
super().__init__(secret_manager_name="in_memory_secrets")
# Store your secrets in memory
print("INITIALIZING CUSTOM SECRET MANAGER IN MEMORY")
self.secrets = {}
print("CUSTOM SECRET MANAGER IN MEMORY INITIALIZED")
async def async_read_secret(
self,
secret_name: str,
optional_params: Optional[dict] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Optional[str]:
"""Read secret asynchronously"""
print("READING SECRET ASYNCHRONOUSLY")
print("SECRET NAME: %s", secret_name)
print("SECRET: %s", self.secrets.get(secret_name))
return self.secrets.get(secret_name)
def sync_read_secret(
self,
secret_name: str,
optional_params: Optional[dict] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Optional[str]:
"""Read secret synchronously"""
from litellm._logging import verbose_proxy_logger
verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}")
value = self.secrets.get(secret_name)
verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: READ SECRET: {value}")
return value
async def async_write_secret(
self,
secret_name: str,
secret_value: str,
description: Optional[str] = None,
optional_params: Optional[dict] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
tags: Optional[Union[dict, list]] = None,
) -> dict:
"""Write a secret to the in-memory store"""
self.secrets[secret_name] = secret_value
print("ALL SECRETS=%s", self.secrets)
return {
"status": "success",
"secret_name": secret_name,
"description": description,
}
async def async_delete_secret(
self,
secret_name: str,
recovery_window_in_days: Optional[int] = 7,
optional_params: Optional[dict] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> dict:
"""Delete a secret from the in-memory store"""
if secret_name in self.secrets:
del self.secrets[secret_name]
return {"status": "deleted", "secret_name": secret_name}
return {"status": "not_found", "secret_name": secret_name}

2
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.6
version: 0.4.7
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

4
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -27,6 +27,10 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
containers:
- name: prisma-migrations

BIN
dist/litellm-1.79.1.tar.gz vendored Normal file
View file

Binary file not shown.

3
docker/Dockerfile.dev
View file

@ -57,6 +57,9 @@ USER root
# Install only runtime dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
libssl3 \
libatomic1 \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app

57
docker/Dockerfile.non_root
View file

@ -8,16 +8,36 @@ ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
# Install build dependencies
# Install build dependencies including Node.js for UI build
USER root
RUN apk add --no-cache build-base bash \
RUN apk add --no-cache build-base bash nodejs npm \
&& pip install --no-cache-dir --upgrade pip build
# Copy project files
COPY . .
# Set LITELLM_NON_ROOT flag for build time
ENV LITELLM_NON_ROOT=true
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
RUN mkdir -p /tmp/litellm_ui && \
cd ui/litellm-dashboard && \
if [ -f "../../enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp ../../enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
npm install && \
npm run build && \
cp -r ./out/* /tmp/litellm_ui/ && \
cd /tmp/litellm_ui && \
for html_file in *.html; do \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
mkdir -p "$folder_name" && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
done && \
cd /app/ui/litellm-dashboard && \
rm -rf ./out
# Build package and wheel dependencies
RUN rm -rf dist/* && python -m build && \
@ -42,12 +62,17 @@ COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
COPY --from=builder /app/schema.prisma /app/schema.prisma
COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui
# Install package from wheel and dependencies
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
&& rm -f *.whl \
&& rm -rf /wheels
# Remove test files and keys from dependencies
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
@ -56,7 +81,6 @@ RUN pip uninstall jwt -y && \
pip uninstall PyJWT -y && \
pip install PyJWT==2.9.0 --no-cache-dir
# --- Prisma Handling for Non-Root User ---
# Set Prisma cache directories
ENV PRISMA_BINARY_CACHE_DIR=/nonexistent
ENV NPM_CONFIG_CACHE=/.npm
@ -68,25 +92,20 @@ RUN pip install --no-cache-dir prisma && \
# Create directories and set permissions for non-root user
RUN mkdir -p /nonexistent /.npm && \
chown -R nobody:nogroup /app && \
chown -R nobody:nogroup /nonexistent /.npm && \
chown -R nobody:nogroup /app /tmp/litellm_ui /nonexistent /.npm && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
# --- OpenShift Compatibility: Apply Red Hat recommended pattern ---
# Get paths for directories that need write access at runtime
# OpenShift compatibility
RUN PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
# Set group ownership to 0 (root group) for OpenShift compatibility && \
chgrp -R 0 $PRISMA_PATH && \
chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
# Mirror owner permissions to group (g=u) as recommended by Red Hat && \
chmod -R g=u $PRISMA_PATH && \
chmod -R g=u $PRISMA_PATH /tmp/litellm_ui && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
# Ensure directories are writable by group && \
chmod -R g+w $PRISMA_PATH && \
chmod -R g+w $PRISMA_PATH /tmp/litellm_ui && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true
# Switch to non-root user
@ -94,14 +113,14 @@ USER nobody
# Set HOME for prisma generate to have a writable directory
ENV HOME=/app
# Set LITELLM_NON_ROOT flag for runtime
ENV LITELLM_NON_ROOT=true
RUN prisma generate
# --- End of Prisma Handling ---
EXPOSE 4000/tcp
# Set entrypoint and command
ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
# CMD ["--port", "4000", "--detailed_debug"]
CMD ["--port", "4000"]
CMD ["--port", "4000"]

412
docs/my-website/docs/adding_provider/adding_guardrail_support.md Normal file
View file

@ -0,0 +1,412 @@
# Adding Guardrail Support to Endpoints
This guide explains how to add guardrail translation support to new LiteLLM endpoints (e.g., Chat Completions, Responses API, etc.).
## When to Add Guardrail Support
Add guardrail support when:
- You're creating a new LiteLLM endpoint (e.g., a new API format)
- You want to enable guardrails for an existing endpoint that doesn't support them
- You need custom text extraction logic for a specific message format
## Directory Structure
Guardrail handlers follow this structure:
```
litellm/llms/{provider}/{endpoint}/guardrail_translation/
├── __init__.py # Exports handler and registers call types
├── handler.py # Main handler implementation
└── README.md # Documentation (optional but recommended)
```
### Example Structures
**OpenAI Chat Completions:**
```
litellm/llms/openai/chat/guardrail_translation/
├── __init__.py
├── handler.py
└── README.md
```
**OpenAI Responses API:**
```
litellm/llms/openai/responses/guardrail_translation/
├── __init__.py
├── handler.py
└── README.md
```
**Anthropic Messages:**
```
litellm/llms/anthropic/chat/guardrail_translation/
├── __init__.py
└── handler.py
```
## Step-by-Step Implementation
### Step 1: Create the Handler Class
Create `handler.py` that inherits from `BaseTranslation`:
```python
"""
{Provider} {Endpoint} Handler for Unified Guardrails
This module provides guardrail translation support for {Provider}'s {Endpoint} format.
"""
import asyncio
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.utils import ModelResponse # Or appropriate response type
class MyEndpointHandler(BaseTranslation):
"""
Handler for processing {Endpoint} with guardrails.
This class provides methods to:
1. Process input (pre-call hook)
2. Process output response (post-call hook)
"""
async def process_input_messages(
self,
data: dict,
guardrail_to_apply: "CustomGuardrail",
) -> Any:
"""
Process input by applying guardrails to text content.
Args:
data: Request data dictionary
guardrail_to_apply: The guardrail instance to apply
Returns:
Modified data with guardrails applied
"""
# Your implementation here
pass
async def process_output_response(
self,
response: Any, # Use appropriate response type
guardrail_to_apply: "CustomGuardrail",
) -> Any:
"""
Process output response by applying guardrails to text content.
Args:
response: API response object
guardrail_to_apply: The guardrail instance to apply
Returns:
Modified response with guardrails applied
"""
# Your implementation here
pass
```
### Step 2: Implement Core Methods
#### A. Process Input Messages
Extract text from input, apply guardrails, and map back:
```python
async def process_input_messages(
self,
data: dict,
guardrail_to_apply: "CustomGuardrail",
) -> Any:
"""Process input messages by applying guardrails to text content."""
# 1. Get input data from request
messages = data.get("messages") # or appropriate field
if messages is None:
return data
# 2. Extract text and create tasks
tasks = []
task_mappings: List[Tuple[int, Optional[int]]] = []
for msg_idx, message in enumerate(messages):
await self._extract_input_text_and_create_tasks(
message=message,
msg_idx=msg_idx,
tasks=tasks,
task_mappings=task_mappings,
guardrail_to_apply=guardrail_to_apply,
)
# 3. Run all guardrail tasks in parallel
if tasks:
responses = await asyncio.gather(*tasks)
# 4. Map responses back to original structure
await self._apply_guardrail_responses_to_input(
messages=messages,
responses=responses,
task_mappings=task_mappings,
)
return data
```
#### B. Process Output Response
Extract text from response, apply guardrails, and update:
```python
async def process_output_response(
self,
response: "ModelResponse",
guardrail_to_apply: "CustomGuardrail",
) -> Any:
"""Process output response by applying guardrails to text content."""
# 1. Check if response has text to process
if not self._has_text_content(response):
return response
# 2. Extract text and create tasks
tasks = []
task_mappings: List[Tuple[int, Optional[int]]] = []
for idx, item in enumerate(response.choices): # or appropriate field
await self._extract_output_text_and_create_tasks(
item=item,
idx=idx,
tasks=tasks,
task_mappings=task_mappings,
guardrail_to_apply=guardrail_to_apply,
)
# 3. Run all guardrail tasks in parallel
if tasks:
responses = await asyncio.gather(*tasks)
# 4. Update response with guardrailed text
await self._apply_guardrail_responses_to_output(
response=response,
responses=responses,
task_mappings=task_mappings,
)
return response
```
### Step 3: Create Helper Methods
Implement helper methods for text extraction and mapping:
```python
async def _extract_input_text_and_create_tasks(
self,
message: Dict[str, Any],
msg_idx: int,
tasks: List,
task_mappings: List[Tuple[int, Optional[int]]],
guardrail_to_apply: "CustomGuardrail",
) -> None:
"""Extract text content from a message and create guardrail tasks."""
content = message.get("content")
if content is None:
return
if isinstance(content, str):
# Simple string content
tasks.append(guardrail_to_apply.apply_guardrail(text=content))
task_mappings.append((msg_idx, None))
elif isinstance(content, list):
# List content (e.g., multimodal)
for content_idx, content_item in enumerate(content):
if isinstance(content_item, dict):
text_str = content_item.get("text")
if text_str:
tasks.append(guardrail_to_apply.apply_guardrail(text=text_str))
task_mappings.append((msg_idx, int(content_idx)))
async def _apply_guardrail_responses_to_input(
self,
messages: List[Dict[str, Any]],
responses: List[str],
task_mappings: List[Tuple[int, Optional[int]]],
) -> None:
"""Apply guardrail responses back to input messages."""
for task_idx, guardrail_response in enumerate(responses):
msg_idx, content_idx = task_mappings[task_idx]
if content_idx is None:
# String content
messages[msg_idx]["content"] = guardrail_response
else:
# List content
messages[msg_idx]["content"][content_idx]["text"] = guardrail_response
def _has_text_content(self, response: Any) -> bool:
"""Check if response has any text content to process."""
# Implement based on your response structure
return True # or appropriate logic
```
### Step 4: Register the Handler
Create `__init__.py` to register the handler with call types:
```python
"""My Endpoint handler for Unified Guardrails."""
from litellm.llms.{provider}/{endpoint}/guardrail_translation.handler import (
MyEndpointHandler,
)
from litellm.types.utils import CallTypes
guardrail_translation_mappings = {
CallTypes.my_endpoint: MyEndpointHandler,
CallTypes.amy_endpoint: MyEndpointHandler, # async version if applicable
}
__all__ = ["guardrail_translation_mappings"]
```
**Important:** Make sure your `CallTypes` are defined in `litellm/types/utils.py`.
### Step 5: Add Documentation
Create `README.md` with usage examples and format details:
```markdown
# {Provider} {Endpoint} Guardrail Translation Handler
Handler for processing {Provider}'s {Endpoint} with guardrails.
## Overview
This handler processes {Endpoint} input/output by:
1. Extracting text from messages/responses
2. Applying guardrails to text content
3. Mapping guardrailed text back to original structure
## Data Format
### Input Format
```json
{
"field": "value",
"messages": [...]
}
```
### Output Format
```json
{
"field": "value",
"output": [...]
}
```
## Usage
The handler is automatically discovered and applied when guardrails are used with this endpoint.
```bash
curl -X POST 'http://localhost:4000/{my_endpoint}' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer your-api-key' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}],
"guardrails": ["test"]
}'
```
## Extension
Override these methods to customize behavior:
- `_extract_input_text_and_create_tasks()`: Custom text extraction
- `_apply_guardrail_responses_to_input()`: Custom response mapping
- `_has_text_content()`: Custom content detection
```
### Step 6: Add Unit Tests
Create comprehensive tests in `tests/test_litellm/llms/{provider}/{endpoint}/`:
```python
"""
Unit tests for {Provider} {Endpoint} Guardrail Translation Handler
"""
import os
import sys
import pytest
sys.path.insert(0, os.path.abspath("../../../../../.."))
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.llms import get_guardrail_translation_mapping
from litellm.llms.{provider}.{endpoint}.guardrail_translation.handler import (
MyEndpointHandler,
)
from litellm.types.utils import CallTypes
class MockGuardrail(CustomGuardrail):
"""Mock guardrail for testing"""
async def apply_guardrail(self, text: str) -> str:
return f"{text} [GUARDRAILED]"
class TestHandlerDiscovery:
"""Test that the handler is properly discovered"""
def test_handler_discovered(self):
handler_class = get_guardrail_translation_mapping(CallTypes.my_endpoint)
assert handler_class == MyEndpointHandler
class TestInputProcessing:
"""Test input processing functionality"""
@pytest.mark.asyncio
async def test_process_simple_input(self):
handler = MyEndpointHandler()
guardrail = MockGuardrail(guardrail_name="test")
data = {"messages": [{"role": "user", "content": "Hello"}]}
result = await handler.process_input_messages(data, guardrail)
assert result["messages"][0]["content"] == "Hello [GUARDRAILED]"
class TestOutputProcessing:
"""Test output processing functionality"""
@pytest.mark.asyncio
async def test_process_simple_output(self):
handler = MyEndpointHandler()
guardrail = MockGuardrail(guardrail_name="test")
# Create mock response
response = create_mock_response()
result = await handler.process_output_response(response, guardrail)
# Assert guardrail was applied
assert "GUARDRAILED" in get_response_text(result)
```
## Support
For questions or issues:
- Check existing handler implementations for examples
- Review the base translation class documentation
- Create an issue on GitHub with the `guardrails` label

184
docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md Normal file
View file

@ -0,0 +1,184 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Adding a New Guardrail Integration
You're going to create a class that checks text before it goes to the LLM or after it comes back. If it violates your rules, you block it.
## How It Works
Request with guardrail:
```bash
curl --location 'http://localhost:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "How do I hack a system?"}],
"guardrails": ["my-guardrail"]
}'
```
Your guardrail checks input, then output. If something's wrong, raise an exception.
## Build Your Guardrail
### Create Your Directory
```bash
mkdir -p litellm/proxy/guardrails/guardrail_hooks/my_guardrail
cd litellm/proxy/guardrails/guardrail_hooks/my_guardrail
```
Two files: `my_guardrail.py` (main class) and `__init__.py` (initialization).
### Write the Main Class
`my_guardrail.py`:
```python
import os
from typing import Optional, List
from fastapi import HTTPException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.guardrails import PiiEntityType
from litellm._logging import verbose_proxy_logger
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
class MyGuardrail(CustomGuardrail):
def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None, **kwargs):
self.api_key = api_key or os.getenv("MY_GUARDRAIL_API_KEY")
self.api_base = api_base or os.getenv("MY_GUARDRAIL_API_BASE", "https://api.myguardrail.com")
super().__init__(default_on=True)
async def apply_guardrail(
self,
text: str,
language: Optional[str] = None,
entities: Optional[List[PiiEntityType]] = None,
request_data: Optional[dict] = None,
) -> str:
result = await self._check_with_api(text, request_data)
if result.get("action") == "BLOCK":
raise Exception(f"Content blocked: {result.get('reason', 'Policy violation')}")
return text
async def _check_with_api(self, text: str, request_data: Optional[dict]) -> dict:
async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
response = await async_client.post(
f"{self.api_base}/check",
headers=headers,
json={"text": text},
timeout=5,
)
response.raise_for_status()
return response.json()
```
### Create the Init File
`__init__.py`:
```python
from typing import TYPE_CHECKING
from litellm.types.guardrails import SupportedGuardrailIntegrations
from .my_guardrail import MyGuardrail
if TYPE_CHECKING:
from litellm.types.guardrails import Guardrail, LitellmParams
def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"):
import litellm
_my_guardrail_callback = MyGuardrail(
api_base=litellm_params.api_base,
api_key=litellm_params.api_key,
guardrail_name=guardrail.get("guardrail_name", ""),
event_hook=litellm_params.mode,
default_on=litellm_params.default_on,
)
litellm.logging_callback_manager.add_litellm_callback(_my_guardrail_callback)
return _my_guardrail_callback
guardrail_initializer_registry = {
SupportedGuardrailIntegrations.MY_GUARDRAIL.value: initialize_guardrail,
}
guardrail_class_registry = {
SupportedGuardrailIntegrations.MY_GUARDRAIL.value: MyGuardrail,
}
```
### Register Your Guardrail Type
Add to `litellm/types/guardrails.py`:
```python
class SupportedGuardrailIntegrations(str, Enum):
LAKERA = "lakera_prompt_injection"
APORIA = "aporia"
BEDROCK = "bedrock_guardrails"
PRESIDIO = "presidio"
ZSCALER_AI_GUARD = "zscaler_ai_guard"
MY_GUARDRAIL = "my_guardrail"
```
## Usage
### Config File
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
guardrails:
- guardrail_name: my_guardrail
litellm_params:
guardrail: my_guardrail
mode: during_call
api_key: os.environ/MY_GUARDRAIL_API_KEY
api_base: https://api.myguardrail.com
```
### Per-Request
```bash
curl --location 'http://localhost:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Test message"}],
"guardrails": ["my_guardrail"]
}'
```
## Testing
Add unit tests inside `test_litellm/` folder.

11
docs/my-website/docs/anthropic_unified.md
View file

@ -10,13 +10,14 @@ Use LiteLLM to call all your LLM APIs in the Anthropic `v1/messages` format.
| Feature | Supported | Notes |
|-------|-------|-------|
| Cost Tracking | ✅ | |
| Logging | ✅ | works across all integrations |
| Cost Tracking | ✅ | Works with all supported models |
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Streaming | ✅ | |
| Fallbacks | ✅ | between supported models |
| Loadbalancing | ✅ | between supported models |
| Support llm providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai`, etc. |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input and output text (non-streaming only) |
| Supported Providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai`, etc. |
## Usage
---

88
docs/my-website/docs/apply_guardrail.md
View file

@ -3,13 +3,49 @@ import TabItem from '@theme/TabItem';
# /guardrails/apply_guardrail
Use this endpoint to directly call a guardrail configured on your LiteLLM instance. This is useful when you have services that need to directly call a guardrail.
Use this endpoint to directly call a guardrail configured on your LiteLLM instance. This is useful when you have services that need to directly call a guardrail.
## Supported Guardrail Types
This endpoint supports various guardrail types including:
- **Presidio** - PII detection and masking
- **Bedrock** - AWS Bedrock guardrails for content moderation
- **Lakera** - AI safety guardrails
- **Custom guardrails** - User-defined guardrails
## Configuration
### Bedrock Guardrail Configuration
To use Bedrock guardrails with the apply_guardrail endpoint, configure your guardrail in your LiteLLM config.yaml:
```yaml
guardrails:
- guardrail_name: "bedrock-content-guard"
litellm_params:
guardrail: bedrock
mode: "pre_call"
guardrailIdentifier: "your-guardrail-id" # Your actual Bedrock guardrail ID
guardrailVersion: "DRAFT" # or your version number
aws_region_name: "us-east-1" # Your AWS region
aws_role_name: "your-role-arn" # Your AWS role with Bedrock permissions
default_on: true
```
**Required AWS Setup:**
1. Create a Bedrock guardrail in AWS Console
2. Get the guardrail ID and version
3. Ensure your AWS credentials have Bedrock permissions
4. Configure the guardrail in your LiteLLM config
## Usage
---
In this example `mask_pii` is the guardrail name configured on LiteLLM.
<Tabs>
<TabItem value="presidio" label="Presidio PII Guardrail" default>
In this example `mask_pii` is a Presidio guardrail configured on LiteLLM.
```bash showLineNumbers title="Example calling the endpoint"
curl -X POST 'http://localhost:4000/guardrails/apply_guardrail' \
@ -23,6 +59,27 @@ curl -X POST 'http://localhost:4000/guardrails/apply_guardrail' \
}'
```
</TabItem>
<TabItem value="bedrock" label="Bedrock Guardrail">
In this example `bedrock-content-guard` is a Bedrock guardrail configured on LiteLLM.
```bash showLineNumbers title="Example calling the endpoint"
curl -X POST 'http://localhost:4000/guardrails/apply_guardrail' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer your-api-key' \
-d '{
"guardrail_name": "bedrock-content-guard",
"text": "This is potentially harmful content that should be blocked",
"language": "en"
}'
```
**Note**: For Bedrock guardrails, the `entities` parameter is not used as Bedrock handles content moderation based on its own policies.
</TabItem>
</Tabs>
## Request Format
---
@ -59,12 +116,39 @@ The response will contain the processed text after applying the guardrail.
#### Example Response
<Tabs>
<TabItem value="presidio" label="Presidio Response" default>
```json
{
"response_text": "My name is [REDACTED] and my email is [REDACTED]"
}
```
</TabItem>
<TabItem value="bedrock" label="Bedrock Response">
```json
{
"response_text": "This is potentially harmful content that should be blocked"
}
```
**Note**: If Bedrock guardrail blocks the content, the endpoint will return an error with the blocking reason.
</TabItem>
</Tabs>
#### Response Fields
- **response_text** (string):
The text after applying the guardrail.
#### Error Responses
If a guardrail blocks content (e.g., Bedrock guardrail), the endpoint will return an error:
```json
{
"detail": "Content blocked by Bedrock guardrail: Content violates policy"
}
```

11
docs/my-website/docs/audio_transcription.md
View file

@ -7,12 +7,13 @@ import TabItem from '@theme/TabItem';
| Feature | Supported | Notes |
|-------|-------|-------|
| Cost Tracking | ✅ | |
| Logging | ✅ | works across all integrations |
| Cost Tracking | ✅ | Works with all supported models |
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Fallbacks | ✅ | between supported models |
| Loadbalancing | ✅ | between supported models |
| Support llm providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai` | |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to output transcribed text (non-streaming only) |
| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai` | |
## Quick Start

151
docs/my-website/docs/bedrock_converse.md Normal file
View file

@ -0,0 +1,151 @@
# /converse
Call Bedrock's `/converse` endpoint through LiteLLM Proxy.
| Feature | Supported |
|---------|-----------|
| Cost Tracking | ✅ |
| Logging | ✅ |
| Streaming | ✅ via `/converse-stream` |
| Load Balancing | ✅ |
## Quick Start
### 1. Setup config.yaml
```yaml showLineNumbers
model_list:
- model_name: my-bedrock-model
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-west-2
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # reads from environment
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
custom_llm_provider: bedrock
```
Set AWS credentials in your environment:
```bash showLineNumbers
export AWS_ACCESS_KEY_ID="your-access-key"
export AWS_SECRET_ACCESS_KEY="your-secret-key"
```
### 2. Start Proxy
```bash showLineNumbers
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Call /converse endpoint
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"messages": [
{
"role": "user",
"content": [{"text": "Hello, how are you?"}]
}
],
"inferenceConfig": {
"temperature": 0.5,
"maxTokens": 100
}
}'
```
## Streaming
For streaming responses, use `/converse-stream`:
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse-stream' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"messages": [
{
"role": "user",
"content": [{"text": "Tell me a short story"}]
}
],
"inferenceConfig": {
"temperature": 0.7,
"maxTokens": 200
}
}'
```
## Load Balancing
Define multiple deployments with the same `model_name` for automatic load balancing:
```yaml showLineNumbers
model_list:
# Deployment 1 - us-west-2
- model_name: my-bedrock-model
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-west-2
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
custom_llm_provider: bedrock
# Deployment 2 - us-east-1
- model_name: my-bedrock-model
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-east-1
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
custom_llm_provider: bedrock
```
The proxy automatically distributes requests across both regions.
## Using boto3 SDK
```python showLineNumbers
import boto3
import json
import os
# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy)
os.environ['AWS_ACCESS_KEY_ID'] = 'dummy'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'dummy'
os.environ['AWS_BEARER_TOKEN_BEDROCK'] = "sk-1234" # your litellm proxy api key
# Point boto3 to the LiteLLM proxy
bedrock_runtime = boto3.client(
service_name='bedrock-runtime',
region_name='us-west-2',
endpoint_url='http://0.0.0.0:4000/bedrock'
)
response = bedrock_runtime.converse(
modelId='my-bedrock-model', # Your model_name from config.yaml
messages=[
{
"role": "user",
"content": [{"text": "Hello, how are you?"}]
}
],
inferenceConfig={
"temperature": 0.5,
"maxTokens": 100
}
)
print(response['output']['message']['content'][0]['text'])
```
## More Info
For complete documentation including Guardrails, Knowledge Bases, and Agents, see:
- [Full Bedrock Passthrough Docs](./pass_through/bedrock)

145
docs/my-website/docs/bedrock_invoke.md Normal file
View file

@ -0,0 +1,145 @@
# /invoke
Call Bedrock's `/invoke` endpoint through LiteLLM Proxy.
| Feature | Supported |
|---------|-----------|
| Cost Tracking | ✅ |
| Logging | ✅ |
| Streaming | ✅ via `/invoke-with-response-stream` |
| Load Balancing | ✅ |
## Quick Start
### 1. Setup config.yaml
```yaml showLineNumbers
model_list:
- model_name: my-bedrock-model
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-west-2
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # reads from environment
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
custom_llm_provider: bedrock
```
Set AWS credentials in your environment:
```bash showLineNumbers
export AWS_ACCESS_KEY_ID="your-access-key"
export AWS_SECRET_ACCESS_KEY="your-secret-key"
```
### 2. Start Proxy
```bash showLineNumbers
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Call /invoke endpoint
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/invoke' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"max_tokens": 100,
"messages": [
{
"role": "user",
"content": "Hello, how are you?"
}
],
"anthropic_version": "bedrock-2023-05-31"
}'
```
## Streaming
For streaming responses, use `/invoke-with-response-stream`:
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/invoke-with-response-stream' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"max_tokens": 100,
"messages": [
{
"role": "user",
"content": "Tell me a short story"
}
],
"anthropic_version": "bedrock-2023-05-31"
}'
```
## Load Balancing
Define multiple deployments with the same `model_name` for automatic load balancing:
```yaml showLineNumbers
model_list:
# Deployment 1 - us-west-2
- model_name: my-bedrock-model
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-west-2
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
custom_llm_provider: bedrock
# Deployment 2 - us-east-1
- model_name: my-bedrock-model
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-east-1
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
custom_llm_provider: bedrock
```
The proxy automatically distributes requests across both regions.
## Using boto3 SDK
```python showLineNumbers
import boto3
import json
import os
# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy)
os.environ['AWS_ACCESS_KEY_ID'] = 'dummy'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'dummy'
os.environ['AWS_BEARER_TOKEN_BEDROCK'] = "sk-1234" # your litellm proxy api key
# Point boto3 to the LiteLLM proxy
bedrock_runtime = boto3.client(
service_name='bedrock-runtime',
region_name='us-west-2',
endpoint_url='http://0.0.0.0:4000/bedrock'
)
response = bedrock_runtime.invoke_model(
modelId='my-bedrock-model', # Your model_name from config.yaml
contentType='application/json',
accept='application/json',
body=json.dumps({
"max_tokens": 100,
"messages": [{"role": "user", "content": "Hello"}],
"anthropic_version": "bedrock-2023-05-31"
})
)
response_body = json.loads(response['body'].read())
print(response_body['content'][0]['text'])
```
## More Info
For complete documentation including Guardrails, Knowledge Bases, and Agents, see:
- [Full Bedrock Passthrough Docs](./pass_through/bedrock)

46
docs/my-website/docs/benchmarks.md
View file

@ -55,6 +55,10 @@ Each machine deploying LiteLLM had the following specs:
- 4 CPU
- 8GB RAM
## Configuration
- Database: PostgreSQL
- Redis: Not used
## Locust Settings
@ -118,6 +122,48 @@ class MyUser(HttpUser):
```
## LiteLLM vs Portkey Performance Comparison
**Test Configuration**: 4 CPUs, 8 GB RAM per instance | Load: 1k concurrent users, 500 ramp-up
### Multi-Instance (4×) Performance
| Metric | Portkey (no DB) | LiteLLM (with DB) |
| ------------------- | --------------- | ----------------- |
| **Total Requests** | 293,796 | 312,405 |
| **Failed Requests** | 0 | 0 |
| **Median Latency** | 100 ms | 100 ms |
| **p95 Latency** | 230 ms | 150 ms |
| **p99 Latency** | 500 ms | 240 ms |
| **Average Latency** | 123 ms | 111 ms |
| **Current RPS** | 1,170.9 | 1,170 |
### Technical Insights
**Portkey**
**Pros**
* Low memory footprint
* Stable latency with minimal spikes
**Cons**
* CPU utilization capped around ~40%, indicating underutilization of available compute resources
* Experienced three I/O timeout outages
**LiteLLM**
**Pros**
* Fully utilizes available CPU capacity
* Strong connection handling and low latency after initial warm-up spikes
**Cons**
* High memory usage during initialization and per request
## Logging Callbacks

74
docs/my-website/docs/completion/image_generation_chat.md
View file

@ -15,16 +15,22 @@ Supported Providers:
- Google AI Studio (`gemini`)
- Vertex AI (`vertex_ai/`)
LiteLLM will standardize the `image` response in the assistant message for models that support image generation during chat completions.
LiteLLM will standardize the `images` response in the assistant message for models that support image generation during chat completions.
```python title="Example response from litellm"
"message": {
...
"content": "Here's the image you requested:",
"image": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
"images": [
{
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
},
"index": 0,
"type": "image_url"
}
]
}
```
@ -47,7 +53,7 @@ response = completion(
)
print(response.choices[0].message.content) # Text response
print(response.choices[0].message.image) # Image data
print(response.choices[0].message.images) # List of image objects
```
</TabItem>
@ -103,10 +109,16 @@ curl http://0.0.0.0:4000/v1/chat/completions \
"message": {
"content": "Here's the image you requested:",
"role": "assistant",
"image": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
"images": [
{
"image_url": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
},
"index": 0,
"type": "image_url"
}
]
}
}
],
@ -141,8 +153,8 @@ response = completion(
)
for chunk in response:
if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None:
print("Generated image:", chunk.choices[0].delta.image["url"])
if hasattr(chunk.choices[0].delta, "images") and chunk.choices[0].delta.images is not None:
print("Generated image:", chunk.choices[0].delta.images[0]["image_url"]["url"])
break
```
@ -175,7 +187,7 @@ data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"content":"Here's the image you requested:"},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"image":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...","detail":"auto"}},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"images":[{"image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...","detail":"auto"},"index":0,"type":"image_url"}]},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
@ -200,8 +212,8 @@ async def generate_image():
)
print(response.choices[0].message.content) # Text response
print(response.choices[0].message.image) # Image data
print(response.choices[0].message.images) # List of image objects
return response
# Run the async function
@ -212,21 +224,31 @@ asyncio.run(generate_image())
| Provider | Model |
|----------|--------|
| Google AI Studio | `gemini/gemini-2.5-flash-image-preview` |
| Vertex AI | `vertex_ai/gemini-2.5-flash-image-preview` |
| Google AI Studio | `gemini/gemini-2.0-flash-preview-image-generation`, `gemini/gemini-2.5-flash-image-preview` |
| Vertex AI | `vertex_ai/gemini-2.0-flash-preview-image-generation`, `vertex_ai/gemini-2.5-flash-image-preview` |
## Spec
## Spec
The `image` field in the response follows this structure:
The `images` field in the response follows this structure:
```python
"image": {
"url": "data:image/png;base64,<base64_encoded_image>",
"detail": "auto"
}
"images": [
{
"image_url": {
"url": "data:image/png;base64,<base64_encoded_image>",
"detail": "auto"
},
"index": 0,
"type": "image_url"
}
]
```
- `url` - str: Base64 encoded image data in data URI format
- `detail` - str: Image detail level (always "auto" for generated images)
- `images` - List[ImageURLListItem]: Array of generated images
- `image_url` - ImageURLObject: Container for image data
- `url` - str: Base64 encoded image data in data URI format
- `detail` - str: Image detail level (always "auto" for generated images)
- `index` - int: Index of the image in the response
- `type` - str: Type identifier (always "image_url")
The image is returned as a base64-encoded data URI that can be directly used in HTML `<img>` tags or saved to a file.
The images are returned as base64-encoded data URIs that can be directly used in HTML `<img>` tags or saved to files.

45
docs/my-website/docs/completion/json_mode.md
View file

@ -309,33 +309,30 @@ curl http://0.0.0.0:4000/v1/chat/completions \
{"role": "user", "content": "Alice and Bob are going to a science fair on Friday."},
],
"response_format": {
"type": "json_object",
"response_schema": {
"type": "json_schema",
"json_schema": {
"name": "math_reasoning",
"schema": {
"type": "object",
"properties": {
"steps": {
"type": "array",
"items": {
"type": "object",
"properties": {
"explanation": { "type": "string" },
"output": { "type": "string" }
},
"required": ["explanation", "output"],
"additionalProperties": false
}
"type": "json_schema",
"json_schema": {
"name": "math_reasoning",
"schema": {
"type": "object",
"properties": {
"steps": {
"type": "array",
"items": {
"type": "object",
"properties": {
"explanation": { "type": "string" },
"output": { "type": "string" }
},
"final_answer": { "type": "string" }
},
"required": ["steps", "final_answer"],
"additionalProperties": false
"required": ["explanation", "output"],
"additionalProperties": false
}
},
"strict": true
"final_answer": { "type": "string" }
},
"required": ["steps", "final_answer"],
"additionalProperties": false
},
"strict": true
}
},
}'

215
docs/my-website/docs/completion/knowledgebase.md
View file

@ -18,7 +18,7 @@ LiteLLM integrates with vector stores, allowing your models to access your organ
## Supported Vector Stores
- [Bedrock Knowledge Bases](https://aws.amazon.com/bedrock/knowledge-bases/)
- [OpenAI Vector Stores](https://platform.openai.com/docs/api-reference/vector-stores/search)
- [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores)
- [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores) (Cannot be directly queried. Only available for calling in Assistants messages. We will be adding Azure AI Search Vector Store API support soon.)
- [Vertex AI RAG API](https://cloud.google.com/vertex-ai/generative-ai/docs/rag-overview)
## Quick Start
@ -412,6 +412,219 @@ This is sent to: `https://bedrock-agent-runtime.{aws_region}.amazonaws.com/knowl
This process happens automatically whenever you include the `vector_store_ids` parameter in your request.
## Accessing Search Results (Citations)
When using vector stores, LiteLLM automatically returns search results in `provider_specific_fields`. This allows you to show users citations for the AI's response.
### Key Concept
Search results are always in: `response.choices[0].message.provider_specific_fields["search_results"]`
For streaming: Results appear in the **final chunk** when `finish_reason == "stop"`
### Non-Streaming Example
**Non-Streaming Response with search results:**
```json
{
"id": "chatcmpl-abc123",
"choices": [{
"index": 0,
"message": {
"role": "assistant",
"content": "LiteLLM is a platform...",
"provider_specific_fields": {
"search_results": [{
"search_query": "What is litellm?",
"data": [{
"score": 0.95,
"content": [{"text": "...", "type": "text"}],
"filename": "litellm-docs.md",
"file_id": "doc-123"
}]
}]
}
},
"finish_reason": "stop"
}]
}
```
<Tabs>
<TabItem value="python-sdk" label="Python SDK">
```python
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-litellm-api-key"
)
response = client.chat.completions.create(
model="claude-3-5-sonnet",
messages=[{"role": "user", "content": "What is litellm?"}],
tools=[{"type": "file_search", "vector_store_ids": ["T37J8R4WTM"]}]
)
# Get AI response
print(response.choices[0].message.content)
# Get search results (citations)
search_results = response.choices[0].message.provider_specific_fields.get("search_results", [])
for result_page in search_results:
for idx, item in enumerate(result_page['data'], 1):
print(f"[{idx}] {item.get('filename', 'Unknown')} (score: {item['score']:.2f})")
```
</TabItem>
<TabItem value="typescript" label="TypeScript SDK">
```typescript
import OpenAI from 'openai';
const client = new OpenAI({
baseURL: 'http://localhost:4000',
apiKey: process.env.LITELLM_API_KEY
});
const response = await client.chat.completions.create({
model: 'claude-3-5-sonnet',
messages: [{ role: 'user', content: 'What is litellm?' }],
tools: [{ type: 'file_search', vector_store_ids: ['T37J8R4WTM'] }]
});
// Get AI response
console.log(response.choices[0].message.content);
// Get search results (citations)
const message = response.choices[0].message as any;
const searchResults = message.provider_specific_fields?.search_results || [];
searchResults.forEach((page: any) => {
page.data.forEach((item: any, idx: number) => {
console.log(`[${idx + 1}] ${item.filename || 'Unknown'} (${item.score.toFixed(2)})`);
});
});
```
</TabItem>
</Tabs>
### Streaming Example
**Streaming Response with search results (final chunk):**
```json
{
"id": "chatcmpl-abc123",
"choices": [{
"index": 0,
"delta": {
"provider_specific_fields": {
"search_results": [{
"search_query": "What is litellm?",
"data": [{
"score": 0.95,
"content": [{"text": "...", "type": "text"}],
"filename": "litellm-docs.md",
"file_id": "doc-123"
}]
}]
}
},
"finish_reason": "stop"
}]
}
```
<Tabs>
<TabItem value="python-sdk" label="Python SDK">
```python
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-litellm-api-key"
)
stream = client.chat.completions.create(
model="claude-3-5-sonnet",
messages=[{"role": "user", "content": "What is litellm?"}],
tools=[{"type": "file_search", "vector_store_ids": ["T37J8R4WTM"]}],
stream=True
)
for chunk in stream:
# Stream content
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="", flush=True)
# Get citations in final chunk
if chunk.choices[0].finish_reason == "stop":
search_results = getattr(chunk.choices[0].delta, 'provider_specific_fields', {}).get('search_results', [])
if search_results:
print("\n\nSources:")
for page in search_results:
for idx, item in enumerate(page['data'], 1):
print(f" [{idx}] {item.get('filename', 'Unknown')} ({item['score']:.2f})")
```
</TabItem>
<TabItem value="typescript" label="TypeScript SDK">
```typescript
import OpenAI from 'openai';
const stream = await client.chat.completions.create({
model: 'claude-3-5-sonnet',
messages: [{ role: 'user', content: 'What is litellm?' }],
tools: [{ type: 'file_search', vector_store_ids: ['T37J8R4WTM'] }],
stream: true
});
for await (const chunk of stream) {
// Stream content
if (chunk.choices[0]?.delta?.content) {
process.stdout.write(chunk.choices[0].delta.content);
}
// Get citations in final chunk
if (chunk.choices[0]?.finish_reason === 'stop') {
const searchResults = (chunk.choices[0].delta as any).provider_specific_fields?.search_results || [];
if (searchResults.length > 0) {
console.log('\n\nSources:');
searchResults.forEach((page: any) => {
page.data.forEach((item: any, idx: number) => {
console.log(` [${idx + 1}] ${item.filename || 'Unknown'} (${item.score.toFixed(2)})`);
});
});
}
}
}
```
</TabItem>
</Tabs>
### Search Result Fields
| Field | Type | Description |
|-------|------|-------------|
| `search_query` | string | The query used to search the vector store |
| `data` | array | Array of search results |
| `data[].score` | float | Relevance score (0-1, higher is more relevant) |
| `data[].content` | array | Content chunks with `text` and `type` |
| `data[].filename` | string | Name of the source file (optional) |
| `data[].file_id` | string | Identifier for the source file (optional) |
| `data[].attributes` | object | Provider-specific metadata (optional) |
## API Reference
### LiteLLM Completion Knowledge Base Parameters

10
docs/my-website/docs/completion/prompt_caching.md
View file

@ -27,7 +27,7 @@ For the supported providers, LiteLLM follows the OpenAI prompt caching usage obj
}
```
- `prompt_tokens`: These are the non-cached prompt tokens (same as Anthropic, equivalent to Deepseek `prompt_cache_miss_tokens`).
- `prompt_tokens`: These are all prompt tokens including cache-miss and cache-hit input tokens.
- `completion_tokens`: These are the output tokens generated by the model.
- `total_tokens`: Sum of prompt_tokens + completion_tokens.
- `prompt_tokens_details`: Object containing cached_tokens.
@ -506,3 +506,11 @@ curl -L -X GET 'http://0.0.0.0:4000/v1/model/info' \
</Tabs>
This checks our maintained [model info/cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
## Read More
:::tip Auto-Inject Prompt Caching
Want LiteLLM to automatically add `cache_control` directives without modifying your code?
See [**Auto-Inject Prompt Caching Tutorial**](../tutorials/prompt_caching.md) to learn how to use `cache_control_injection_points` to automatically cache system messages, specific messages by index, or custom injection patterns.
:::

2
docs/my-website/docs/contact.md
View file

@ -2,6 +2,6 @@
[![](https://dcbadge.vercel.app/api/server/wuPM9dRgDw)](https://discord.gg/wuPM9dRgDw)
* [Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
* [Community Slack 💭](https://www.litellm.ai/support)
* [Meet with us 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
* Contact us at ishaan@berri.ai / krrish@berri.ai

465
docs/my-website/docs/containers.md Normal file
View file

@ -0,0 +1,465 @@
# /containers
Manage OpenAI code interpreter containers (sessions) for executing code in isolated environments.
| Feature | Supported |
|---------|-----------|
| Cost Tracking | ✅ |
| Logging | ✅ (Full request/response logging) |
| Load Balancing | ✅ |
| Proxy Server Support | ✅ Full proxy integration with virtual keys |
| Spend Management | ✅ Budget tracking and rate limiting |
| Supported Providers | `openai`|
:::tip
Containers provide isolated execution environments for code interpreter sessions. You can create, list, retrieve, and delete containers.
:::
## **LiteLLM Python SDK Usage**
### Quick Start
**Create a Container**
```python
import litellm
import os
# setup env
os.environ["OPENAI_API_KEY"] = "sk-.."
container = litellm.create_container(
name="My Code Interpreter Container",
custom_llm_provider="openai",
expires_after={
"anchor": "last_active_at",
"minutes": 20
}
)
print(f"Container ID: {container.id}")
print(f"Container Name: {container.name}")
```
### Async Usage
```python
from litellm import acreate_container
import os
os.environ["OPENAI_API_KEY"] = "sk-.."
container = await acreate_container(
name="My Code Interpreter Container",
custom_llm_provider="openai",
expires_after={
"anchor": "last_active_at",
"minutes": 20
}
)
print(f"Container ID: {container.id}")
print(f"Container Name: {container.name}")
```
### List Containers
```python
from litellm import list_containers
import os
os.environ["OPENAI_API_KEY"] = "sk-.."
containers = list_containers(
custom_llm_provider="openai",
limit=20,
order="desc"
)
print(f"Found {len(containers.data)} containers")
for container in containers.data:
print(f" - {container.id}: {container.name}")
```
**Async Usage:**
```python
from litellm import alist_containers
containers = await alist_containers(
custom_llm_provider="openai",
limit=20,
order="desc"
)
print(f"Found {len(containers.data)} containers")
for container in containers.data:
print(f" - {container.id}: {container.name}")
```
### Retrieve a Container
```python
from litellm import retrieve_container
import os
os.environ["OPENAI_API_KEY"] = "sk-.."
container = retrieve_container(
container_id="cntr_123...",
custom_llm_provider="openai"
)
print(f"Container: {container.name}")
print(f"Status: {container.status}")
print(f"Created: {container.created_at}")
```
**Async Usage:**
```python
from litellm import aretrieve_container
container = await aretrieve_container(
container_id="cntr_123...",
custom_llm_provider="openai"
)
print(f"Container: {container.name}")
print(f"Status: {container.status}")
print(f"Created: {container.created_at}")
```
### Delete a Container
```python
from litellm import delete_container
import os
os.environ["OPENAI_API_KEY"] = "sk-.."
result = delete_container(
container_id="cntr_123...",
custom_llm_provider="openai"
)
print(f"Deleted: {result.deleted}")
print(f"Container ID: {result.id}")
```
**Async Usage:**
```python
from litellm import adelete_container
result = await adelete_container(
container_id="cntr_123...",
custom_llm_provider="openai"
)
print(f"Deleted: {result.deleted}")
print(f"Container ID: {result.id}")
```
## **LiteLLM Proxy Usage**
LiteLLM provides OpenAI API compatible container endpoints for managing code interpreter sessions:
- `/v1/containers` - Create and list containers
- `/v1/containers/{container_id}` - Retrieve and delete containers
**Setup**
```bash
$ export OPENAI_API_KEY="sk-..."
$ litellm
# RUNNING on http://0.0.0.0:4000
```
**Custom Provider Specification**
You can specify the custom LLM provider in multiple ways (priority order):
1. Header: `-H "custom-llm-provider: openai"`
2. Query param: `?custom_llm_provider=openai`
3. Request body: `{"custom_llm_provider": "openai", ...}`
4. Defaults to "openai" if not specified
**Create a Container**
```bash
# Default provider (openai)
curl -X POST "http://localhost:4000/v1/containers" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"name": "My Container",
"expires_after": {
"anchor": "last_active_at",
"minutes": 20
}
}'
```
```bash
# Via header
curl -X POST "http://localhost:4000/v1/containers" \
-H "Authorization: Bearer sk-1234" \
-H "custom-llm-provider: openai" \
-H "Content-Type: application/json" \
-d '{
"name": "My Container"
}'
```
```bash
# Via query parameter
curl -X POST "http://localhost:4000/v1/containers?custom_llm_provider=openai" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"name": "My Container"
}'
```
**List Containers**
```bash
curl "http://localhost:4000/v1/containers?limit=20&order=desc" \
-H "Authorization: Bearer sk-1234"
```
**Retrieve a Container**
```bash
curl "http://localhost:4000/v1/containers/cntr_123..." \
-H "Authorization: Bearer sk-1234"
```
**Delete a Container**
```bash
curl -X DELETE "http://localhost:4000/v1/containers/cntr_123..." \
-H "Authorization: Bearer sk-1234"
```
## **Using OpenAI Client with LiteLLM Proxy**
You can use the standard OpenAI Python client to interact with LiteLLM's container endpoints. This provides a familiar interface while leveraging LiteLLM's proxy features.
### Setup
First, configure your OpenAI client to point to your LiteLLM proxy:
```python
from openai import OpenAI
client = OpenAI(
api_key="sk-1234", # Your LiteLLM proxy key
base_url="http://localhost:4000" # LiteLLM proxy URL
)
```
### Create a Container
```python
container = client.containers.create(
name="test-container",
expires_after={
"anchor": "last_active_at",
"minutes": 20
},
extra_body={"custom_llm_provider": "openai"}
)
print(f"Container ID: {container.id}")
print(f"Container Name: {container.name}")
print(f"Created at: {container.created_at}")
```
### List Containers
```python
containers = client.containers.list(
limit=20,
extra_body={"custom_llm_provider": "openai"}
)
print(f"Found {len(containers.data)} containers")
for container in containers.data:
print(f" - {container.id}: {container.name}")
```
### Retrieve a Container
```python
container = client.containers.retrieve(
container_id="cntr_6901d28b3c8881908b702815828a5bde0380b3408aeae8c7",
extra_body={"custom_llm_provider": "openai"}
)
print(f"Container: {container.name}")
print(f"Status: {container.status}")
print(f"Last active: {container.last_active_at}")
```
### Delete a Container
```python
result = client.containers.delete(
container_id="cntr_6901d28b3c8881908b702815828a5bde0380b3408aeae8c7",
extra_body={"custom_llm_provider": "openai"}
)
print(f"Deleted: {result.deleted}")
print(f"Container ID: {result.id}")
```
### Complete Workflow Example
Here's a complete example showing the full container management workflow:
```python
from openai import OpenAI
# Initialize client
client = OpenAI(
api_key="sk-1234",
base_url="http://localhost:4000"
)
# 1. Create a container
print("Creating container...")
container = client.containers.create(
name="My Code Interpreter Session",
expires_after={
"anchor": "last_active_at",
"minutes": 20
},
extra_body={"custom_llm_provider": "openai"}
)
container_id = container.id
print(f"Container created. ID: {container_id}")
# 2. List all containers
print("\nListing containers...")
containers = client.containers.list(
extra_body={"custom_llm_provider": "openai"}
)
for c in containers.data:
print(f" - {c.id}: {c.name} (Status: {c.status})")
# 3. Retrieve specific container
print(f"\nRetrieving container {container_id}...")
retrieved = client.containers.retrieve(
container_id=container_id,
extra_body={"custom_llm_provider": "openai"}
)
print(f"Container: {retrieved.name}")
print(f"Status: {retrieved.status}")
print(f"Last active: {retrieved.last_active_at}")
# 4. Delete container
print(f"\nDeleting container {container_id}...")
result = client.containers.delete(
container_id=container_id,
extra_body={"custom_llm_provider": "openai"}
)
print(f"Deleted: {result.deleted}")
```
## Container Parameters
### Create Container Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `name` | string | Yes | Name of the container |
| `expires_after` | object | No | Container expiration settings |
| `expires_after.anchor` | string | No | Anchor point for expiration (e.g., "last_active_at") |
| `expires_after.minutes` | integer | No | Minutes until expiration from anchor |
| `file_ids` | array | No | List of file IDs to include in the container |
| `custom_llm_provider` | string | No | LLM provider to use (default: "openai") |
### List Container Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `after` | string | No | Cursor for pagination |
| `limit` | integer | No | Number of items to return (1-100, default: 20) |
| `order` | string | No | Sort order: "asc" or "desc" (default: "desc") |
| `custom_llm_provider` | string | No | LLM provider to use (default: "openai") |
### Retrieve/Delete Container Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `container_id` | string | Yes | ID of the container to retrieve/delete |
| `custom_llm_provider` | string | No | LLM provider to use (default: "openai") |
## Response Objects
### ContainerObject
```json
{
"id": "cntr_123...",
"object": "container",
"created_at": 1234567890,
"name": "My Container",
"status": "active",
"last_active_at": 1234567890,
"expires_at": 1234569090,
"file_ids": []
}
```
### ContainerListResponse
```json
{
"object": "list",
"data": [
{
"id": "cntr_123...",
"object": "container",
"created_at": 1234567890,
"name": "My Container",
"status": "active"
}
],
"first_id": "cntr_123...",
"last_id": "cntr_456...",
"has_more": false
}
```
### DeleteContainerResult
```json
{
"id": "cntr_123...",
"object": "container.deleted",
"deleted": true
}
```
## **Supported Providers**
| Provider | Support Status | Notes |
|-------------|----------------|-------|
| OpenAI | ✅ Supported | Full support for all container operations |
:::info
Currently, only OpenAI supports container management for code interpreter sessions. Support for additional providers may be added in the future.
:::

79
docs/my-website/docs/exception_mapping.md
View file

@ -112,6 +112,85 @@ except openai.APITimeoutError as e:
print(f"should_retry: {should_retry}")
```
## Advanced
### Accessing Provider-Specific Error Details
LiteLLM exceptions include a `provider_specific_fields` attribute that contains additional error information specific to each provider. This is particularly useful for Azure OpenAI, which provides detailed content filtering information.
#### Azure OpenAI - Content Policy Violation Inner Error Access
When Azure OpenAI returns content policy violations, you can access the detailed content filtering results through the `innererror` field:
```python
import litellm
from litellm.exceptions import ContentPolicyViolationError
try:
response = litellm.completion(
model="azure/gpt-4",
messages=[
{
"role": "user",
"content": "Some content that might violate policies"
}
]
)
except ContentPolicyViolationError as e:
# Access Azure-specific error details
if e.provider_specific_fields and "innererror" in e.provider_specific_fields:
innererror = e.provider_specific_fields["innererror"]
# Access content filter results
content_filter_result = innererror.get("content_filter_result", {})
print(f"Content filter code: {innererror.get('code')}")
print(f"Hate filtered: {content_filter_result.get('hate', {}).get('filtered')}")
print(f"Violence severity: {content_filter_result.get('violence', {}).get('severity')}")
print(f"Sexual content filtered: {content_filter_result.get('sexual', {}).get('filtered')}")
```
**Example Response Structure:**
When calling the LiteLLM proxy, content policy violations will return detailed filtering information:
```json
{
"error": {
"message": "litellm.ContentPolicyViolationError: AzureException - The response was filtered due to the prompt triggering Azure OpenAI's content management policy...",
"type": null,
"param": null,
"code": "400",
"provider_specific_fields": {
"innererror": {
"code": "ResponsibleAIPolicyViolation",
"content_filter_result": {
"hate": {
"filtered": true,
"severity": "high"
},
"jailbreak": {
"filtered": false,
"detected": false
},
"self_harm": {
"filtered": false,
"severity": "safe"
},
"sexual": {
"filtered": false,
"severity": "safe"
},
"violence": {
"filtered": true,
"severity": "medium"
}
}
}
}
}
}
## Details
To see how it's implemented - [check out the code](https://github.com/BerriAI/litellm/blob/a42c197e5a6de56ea576c73715e6c7c6b19fa249/litellm/utils.py#L1217)

206
docs/my-website/docs/extras/creating_adapters.md Normal file
View file

@ -0,0 +1,206 @@
# Call any LiteLLM model in your custom format
Use this to call any LiteLLM supported `.completion()` model, in your custom format. Useful if you have a custom API and want to support any LiteLLM supported model.
## How it works
Your request → Adapter translates to OpenAI format → LiteLLM processes it → Adapter translates response back → Your response
## Create an Adapter
Inherit from `CustomLogger` and implement 3 methods:
```python
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.llms.openai import ChatCompletionRequest
from litellm.types.utils import ModelResponse
class MyAdapter(CustomLogger):
def translate_completion_input_params(self, kwargs) -> ChatCompletionRequest:
"""Convert your format → OpenAI format"""
# Example: Anthropic to OpenAI
return {
"model": kwargs["model"],
"messages": self._convert_messages(kwargs["messages"]),
"max_tokens": kwargs.get("max_tokens"),
}
def translate_completion_output_params(self, response: ModelResponse):
"""Convert OpenAI format → your format"""
# Return your provider's response format
return MyProviderResponse(
id=response.id,
content=response.choices[0].message.content,
usage=response.usage,
)
def translate_completion_output_params_streaming(self, completion_stream):
"""Handle streaming responses"""
return MyStreamWrapper(completion_stream)
```
## Register it
```python
import litellm
my_adapter = MyAdapter()
litellm.adapters = [{"id": "my_provider", "adapter": my_adapter}]
```
## Use it
```python
from litellm import adapter_completion
# Now you can use your provider's format with any LiteLLM model
response = adapter_completion(
adapter_id="my_provider",
model="gpt-4", # or any LiteLLM model
messages=[{"role": "user", "content": "hello"}],
max_tokens=100
)
```
### Streaming
```python
stream = adapter_completion(
adapter_id="my_provider",
model="gpt-4",
messages=[{"role": "user", "content": "hello"}],
stream=True
)
for chunk in stream:
print(chunk)
```
### Async
```python
from litellm import aadapter_completion
response = await aadapter_completion(
adapter_id="my_provider",
model="gpt-4",
messages=[{"role": "user", "content": "hello"}]
)
```
## Example: Anthropic Adapter
Here's how we translate Anthropic's format:
### Input Translation
```python
def translate_completion_input_params(self, kwargs):
model = kwargs.pop("model")
messages = kwargs.pop("messages")
# Convert Anthropic messages to OpenAI format
openai_messages = []
for msg in messages:
if msg["role"] == "user":
openai_messages.append({
"role": "user",
"content": msg["content"]
})
# Handle system message
if "system" in kwargs:
openai_messages.insert(0, {
"role": "system",
"content": kwargs.pop("system")
})
return {
"model": model,
"messages": openai_messages,
**kwargs # pass through other params
}
```
### Output Translation
```python
def translate_completion_output_params(self, response):
return AnthropicResponse(
id=response.id,
type="message",
role="assistant",
content=[{
"type": "text",
"text": response.choices[0].message.content
}],
usage={
"input_tokens": response.usage.prompt_tokens,
"output_tokens": response.usage.completion_tokens
}
)
```
### Streaming
```python
from litellm.types.utils import AdapterCompletionStreamWrapper
class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
def __init__(self, completion_stream, model):
super().__init__(completion_stream)
self.model = model
self.first_chunk = True
async def __anext__(self):
# First chunk
if self.first_chunk:
self.first_chunk = False
return {"type": "message_start", "message": {...}}
# Stream chunks
async for chunk in self.completion_stream:
return {
"type": "content_block_delta",
"delta": {"text": chunk.choices[0].delta.content}
}
# Last chunk
return {"type": "message_stop"}
def translate_completion_output_params_streaming(self, stream, model):
return AnthropicStreamWrapper(stream, model)
```
## Use with Proxy
Add to your proxy config:
```yaml
general_settings:
pass_through_endpoints:
- path: "/v1/messages"
target: "my_module.MyAdapter"
```
Then call it:
```bash
curl http://localhost:4000/v1/messages \
-H "Authorization: Bearer sk-1234" \
-d '{"model": "gpt-4", "messages": [...]}'
```
## Real Example
Check out the full Anthropic adapter:
- [transformation.py](https://github.com/BerriAI/litellm/blob/main/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py)
- [handler.py](https://github.com/BerriAI/litellm/blob/main/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py)
- [streaming_iterator.py](https://github.com/BerriAI/litellm/blob/main/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py)
## That's it
1. Create a class that inherits `CustomLogger`
2. Implement the 3 translation methods
3. Register with `litellm.adapters = [{"id": "...", "adapter": ...}]`
4. Call with `adapter_completion(adapter_id="...")`

10
docs/my-website/docs/files_endpoints.md
View file

@ -57,7 +57,7 @@ client = OpenAI(
client.files.create(
file=wav_data,
purpose="user_data",
extra_body={"custom_llm_provider": "openai"}
extra_headers={"custom-llm-provider": "openai"}
)
```
@ -71,7 +71,7 @@ client = OpenAI(
base_url="http://0.0.0.0:4000/v1"
)
files = client.files.list(extra_body={"custom_llm_provider": "openai"})
files = client.files.list(extra_headers={"custom-llm-provider": "openai"})
print("files=", files)
```
@ -85,7 +85,7 @@ client = OpenAI(
base_url="http://0.0.0.0:4000/v1"
)
file = client.files.retrieve(file_id="file-abc123", extra_body={"custom_llm_provider": "openai"})
file = client.files.retrieve(file_id="file-abc123", extra_headers={"custom-llm-provider": "openai"})
print("file=", file)
```
@ -99,7 +99,7 @@ client = OpenAI(
base_url="http://0.0.0.0:4000/v1"
)
response = client.files.delete(file_id="file-abc123", extra_body={"custom_llm_provider": "openai"})
response = client.files.delete(file_id="file-abc123", extra_headers={"custom-llm-provider": "openai"})
print("delete response=", response)
```
@ -113,7 +113,7 @@ client = OpenAI(
base_url="http://0.0.0.0:4000/v1"
)
content = client.files.content(file_id="file-abc123", extra_body={"custom_llm_provider": "openai"})
content = client.files.content(file_id="file-abc123", extra_headers={"custom-llm-provider": "openai"})
print("content=", content)
```

19
docs/my-website/docs/fine_tuning.md
View file

@ -62,7 +62,7 @@ client = AsyncOpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") # base_u
file_name = "openai_batch_completions.jsonl"
response = await client.files.create(
extra_body={"custom_llm_provider": "azure"}, # tell litellm proxy which provider to use
extra_headers={"custom-llm-provider": "azure"}, # tell litellm proxy which provider to use
file=open(file_name, "rb"),
purpose="fine-tune",
)
@ -73,8 +73,8 @@ response = await client.files.create(
```shell
curl http://localhost:4000/v1/files \
-H "Authorization: Bearer sk-1234" \
-H "custom-llm-provider: azure" \
-F purpose="batch" \
-F custom_llm_provider="azure"\
-F file="@mydata.jsonl"
```
</TabItem>
@ -92,7 +92,7 @@ curl http://localhost:4000/v1/files \
ft_job = await client.fine_tuning.jobs.create(
model="gpt-35-turbo-1106", # Azure OpenAI model you want to fine-tune
training_file="file-abc123", # file_id from create file response
extra_body={"custom_llm_provider": "azure"}, # tell litellm proxy which provider to use
extra_headers={"custom-llm-provider": "azure"}, # tell litellm proxy which provider to use
)
```
</TabItem>
@ -103,8 +103,8 @@ ft_job = await client.fine_tuning.jobs.create(
curl http://localhost:4000/v1/fine_tuning/jobs \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-H "custom-llm-provider: azure" \
-d '{
"custom_llm_provider": "azure",
"model": "gpt-35-turbo-1106",
"training_file": "file-abc123"
}'
@ -215,7 +215,7 @@ curl http://localhost:4000/v1/fine_tuning/jobs \
# cancel specific fine tuning job
cancel_ft_job = await client.fine_tuning.jobs.cancel(
fine_tuning_job_id="123", # fine tuning job id
extra_body={"custom_llm_provider": "azure"}, # tell litellm proxy which provider to use
extra_headers={"custom-llm-provider": "azure"}, # tell litellm proxy which provider to use
)
print("response from cancel ft job={}".format(cancel_ft_job))
@ -228,7 +228,7 @@ print("response from cancel ft job={}".format(cancel_ft_job))
curl -X POST http://localhost:4000/v1/fine_tuning/jobs/ftjob-abc123/cancel \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{"custom_llm_provider": "azure"}'
-H "custom-llm-provider: azure"
```
</TabItem>
@ -242,7 +242,7 @@ curl -X POST http://localhost:4000/v1/fine_tuning/jobs/ftjob-abc123/cancel \
```python
list_ft_jobs = await client.fine_tuning.jobs.list(
extra_query={"custom_llm_provider": "azure"} # tell litellm proxy which provider to use
extra_headers={"custom-llm-provider": "azure"} # tell litellm proxy which provider to use
)
print("list of ft jobs={}".format(list_ft_jobs))
@ -252,9 +252,10 @@ print("list of ft jobs={}".format(list_ft_jobs))
<TabItem value="curl" label="curl">
```shell
curl -X GET 'http://localhost:4000/v1/fine_tuning/jobs?custom_llm_provider=azure' \
curl -X GET 'http://localhost:4000/v1/fine_tuning/jobs' \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234"
-H "Authorization: Bearer sk-1234" \
-H "custom-llm-provider: azure"
```
</TabItem>

2
docs/my-website/docs/generateContent.md
View file

@ -1,7 +1,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Google AI generateContent
# /generateContent
Use LiteLLM to call Google AI's generateContent endpoints for text generation, multimodal interactions, and streaming responses.

44
docs/my-website/docs/guides/security_settings.md
View file

@ -117,10 +117,52 @@ litellm_settings:
```bash
export SSL_CERTIFICATE="/path/to/certificate.pem"
```
</TabItem>
</Tabs>
## 5. Use HTTP_PROXY environment variable
## 5. Configure ECDH Curve for SSL/TLS Performance
The `ssl_ecdh_curve` setting allows you to configure the Elliptic Curve Diffie-Hellman (ECDH) curve used for SSL/TLS key exchange. This is particularly useful for disabling Post-Quantum Cryptography (PQC) to improve performance in environments where PQC is not required.
**Use Case:** Some OpenSSL 3.x systems enable PQC by default, which can slow down TLS handshakes. Setting the ECDH curve to `X25519` disables PQC and can significantly improve connection performance.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
litellm.ssl_ecdh_curve = "X25519" # Disables PQC for better performance
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
litellm_settings:
ssl_ecdh_curve: "X25519"
```
</TabItem>
<TabItem value="env_var" label="Environment Variables">
```bash
export SSL_ECDH_CURVE="X25519"
```
</TabItem>
</Tabs>
**Common Valid Curves:**
- `X25519` - Modern, fast curve (recommended for disabling PQC)
- `prime256v1` - NIST P-256 curve
- `secp384r1` - NIST P-384 curve
- `secp521r1` - NIST P-521 curve
**Note:** If an invalid curve name is provided or if your Python/OpenSSL version doesn't support this feature, LiteLLM will log a warning and continue with default curves.
## 6. Use HTTP_PROXY environment variable
Both httpx and aiohttp libraries use `urllib.request.getproxies` from environment variables. Before client initialization, you may set proxy (and optional SSL_CERT_FILE) by setting the environment variables:

84
docs/my-website/docs/image_edits.md
View file

@ -14,9 +14,9 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Supported operations | Create image edits | Single and multiple images supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | |
| Supported LiteLLM Proxy Versions | 1.71.1+ | |
| Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
| Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)** | Gemini supports the new `gemini-2.5-flash-image` family |
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
@ -149,6 +149,54 @@ for i, image_data in enumerate(response.data):
print(f"Image {i+1}: {image_data.url}")
```
```
</TabItem>
<TabItem value="gemini" label="Gemini">
#### Basic Image Edit
```python showLineNumbers title="Gemini Image Edit"
import base64
import os
from litellm import image_edit
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = image_edit(
model="gemini/gemini-2.5-flash-image",
image=open("original_image.png", "rb"),
prompt="Add aurora borealis to the night sky",
size="1792x1024", # mapped to aspectRatio=16:9 for Gemini
)
edited_image_bytes = base64.b64decode(response.data[0].b64_json)
with open("edited_image.png", "wb") as f:
f.write(edited_image_bytes)
```
#### Multiple Images Edit
```python showLineNumbers title="Gemini Multiple Images Edit"
import base64
import os
from litellm import image_edit
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = image_edit(
model="gemini/gemini-2.5-flash-image",
image=[
open("scene.png", "rb"),
open("style_reference.png", "rb"),
],
prompt="Blend the reference style into the scene while keeping the subject sharp.",
)
for idx, image_obj in enumerate(response.data):
with open(f"gemini_edit_{idx}.png", "wb") as f:
f.write(base64.b64decode(image_obj.b64_json))
```
</TabItem>
</Tabs>
@ -224,6 +272,36 @@ curl -X POST "http://localhost:4000/v1/images/edits" \
-F "response_format=url"
```
```
</TabItem>
<TabItem value="gemini" label="Gemini">
1. Add the Gemini image edit model to your `config.yaml`:
```yaml showLineNumbers title="Gemini Proxy Configuration"
model_list:
- model_name: gemini-image-edit
litellm_params:
model: gemini/gemini-2.5-flash-image
api_key: os.environ/GEMINI_API_KEY
```
2. Start the LiteLLM proxy server:
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
```
3. Make an image edit request (Gemini responses are base64-only):
```bash showLineNumbers title="Gemini Proxy Image Edit"
curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-F "model=gemini-image-edit" \
-F "image=@original_image.png" \
-F "prompt=Add a warm golden-hour glow to the scene" \
-F "size=1024x1024"
```
</TabItem>
</Tabs>

12
docs/my-website/docs/image_generation.md
View file

@ -5,6 +5,18 @@ import TabItem from '@theme/TabItem';
# Image Generations
## Overview
| Feature | Supported | Notes |
|---------|-----------|-------|
| Cost Tracking | ✅ | Works with all supported models |
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input prompts (non-streaming only) |
| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, Xinference, Nscale | |
## Quick Start
### LiteLLM Python SDK

33
docs/my-website/docs/mcp.md
View file

@ -107,6 +107,26 @@ For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport t
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
<br/>
### Static Headers
Sometimes your MCP server needs specific headers on every request. Maybe it's an API key, maybe it's a custom header the server expects. Instead of configuring auth, you can just set them directly.
<Image
img={require('../img/static_headers.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
These headers get sent with every request to the server. That's it.
**When to use this:**
- Your server needs custom headers that don't fit the standard auth patterns
- You want full control over exactly what headers are sent
- You're debugging and need to quickly add headers without changing auth configuration
</TabItem>
<TabItem value="config" label="config.yaml">
@ -175,6 +195,7 @@ mcp_servers:
| `authorization` | `Authorization: <auth_value>` |
- **Extra Headers**: Optional list of additional header names that should be forwarded from client to the MCP server
- **Static Headers**: Optional map of header key/value pairs to include every request to the MCP server.
- **Spec Version**: Optional MCP specification version (defaults to `2025-06-18`)
Examples for each auth type:
@ -217,8 +238,14 @@ mcp_servers:
auth_type: "bearer_token"
auth_value: "ghp_example_token"
extra_headers: ["custom_key", "x-custom-header"] # These headers will be forwarded from client
```
# Example with static headers
my_mcp_server:
url: "https://my-mcp-server.com/mcp"
static_headers: # These headers will be requested to the MCP server
X-API-Key: "abc123"
X-Custom-Header: "some-value"
```
### MCP Aliases
@ -1204,6 +1231,10 @@ mcp_servers:
scopes: ["public_repo", "user:email"]
```
**Note**
In the future, users will only need to specify the `url` of the MCP server.
LiteLLM will automatically resolve the corresponding `authorization_url`, `token_url`, and `registration_url` based on the MCP server metadata (e.g., `.well-known/oauth-authorization-server` or `oauth-protected-resource`).
[**See Claude Code Tutorial**](./tutorials/claude_responses_api#connecting-mcp-servers)
## Using your MCP with client side credentials

15
docs/my-website/docs/moderation.md
View file

@ -22,10 +22,19 @@ response = moderation(
For `/moderations` endpoint, there is **no need to specify `model` in the request or on the litellm config.yaml**
Start litellm proxy server
1. Setup config.yaml
```yaml
model_list:
- model_name: text-moderation-stable
litellm_params:
model: openai/omni-moderation-latest
```
2. Start litellm proxy server
```
litellm
litellm --config /path/to/config.yaml
```
@ -41,7 +50,7 @@ client = OpenAI(api_key="<proxy-api-key>", base_url="http://0.0.0.0:4000")
response = client.moderations.create(
input="hello from litellm",
model="text-moderation-stable" # optional, defaults to `omni-moderation-latest`
model="text-moderation-stable"
)
print(response)

6
docs/my-website/docs/observability/braintrust.md
View file

@ -75,6 +75,12 @@ It is recommended that you include the `project_id` or `project_name` to ensure
You can customize the span name in Braintrust logging by passing `span_name` in the metadata. By default, the span name is set to "Chat Completion".
### Custom Span Attributes
You can customize the span id, root span name and span parents in Braintrust logging by passing `span_id`, `root_span_id` and `span_parents` in the metadata.
`span_parents` should be a string containing a list of span ids, joined by ,
<Tabs>
<TabItem value="sdk" label="SDK">

29
docs/my-website/docs/observability/datadog.md
View file

@ -56,12 +56,32 @@ litellm_settings:
**Step 2**: Set Required env variables for datadog
#### Direct API
Send logs directly to Datadog API:
```shell
DD_API_KEY="5f2d0f310***********" # your datadog API Key
DD_SITE="us5.datadoghq.com" # your datadog base url
DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source. use to differentiate dev vs. prod deployments
```
#### Via DataDog Agent
Send logs through a local DataDog agent (useful for containerized environments):
```shell
DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent
DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518)
DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth)
DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source
```
When `DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for:
- Centralized log shipping in containerized environments
- Reducing direct API calls from multiple services
- Leveraging agent-side processing and filtering
**Step 3**: Start the proxy, make a test request
Start proxy
@ -169,8 +189,10 @@ LiteLLM supports customizing the following Datadog environment variables
| Environment Variable | Description | Default Value | Required |
|---------------------|-------------|---------------|----------|
| `DD_API_KEY` | Your Datadog API key for authentication | None | ✅ Yes |
| `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") | None | ✅ Yes |
| `DD_API_KEY` | Your Datadog API key for authentication (required for direct API, optional for agent) | None | Conditional* |
| `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") (required for direct API) | None | Conditional* |
| `DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No |
| `DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No |
| `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No |
| `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No |
| `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No |
@ -178,3 +200,6 @@ LiteLLM supports customizing the following Datadog environment variables
| `HOSTNAME` | Hostname tag for your logs | "" | ❌ No |
| `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No |
\* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required
\* **Optional when using DataDog Agent**: Set `DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required

34
docs/my-website/docs/observability/opik_integration.md
View file

@ -220,7 +220,41 @@ curl --location --request POST 'http://0.0.0.0:4000/chat/completions' \
}'
```
## Automatic Metadata from API Keys
In some cases, the requester may be unable or unaware of how to add Opik metadata to their requests. To ensure all Opik-related actions are properly tracked, LiteLLM Proxy can automatically associate metadata from a user-specific API key when none is provided in the request.
### How It Works
When you create an API key in LiteLLM Proxy, you can attach Opik-specific metadata to the key itself. This metadata will be automatically applied to all requests made with that key, unless the request explicitly provides its own Opik metadata (which takes precedence).
### Usage
**Step 1: Save Opik Metadata to the corresponding Api Key**
Go to 'Virtual Keys', click on your choosen api key and edit 'Settings'.
Now save the opik metadata as user api key metdata.
<Image img={require('../../img/opik_key_metadata.png')} />
**Step 2: Use the key - Opik metadata is automatically applied**
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-key-from-step-1' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "What's the weather like in Boston today?"
}
]
}'
```
All requests made with this key will automatically be tracked in the "TestProject" Opik project with the specified tags, without requiring the user to pass metadata in each request.
## Support & Talk to Founders

6
docs/my-website/docs/observability/sentry.md
View file

@ -61,6 +61,12 @@ print(response)
These options are useful for high-volume applications where sampling a subset of errors and transactions provides sufficient visibility while managing costs.
#### Sentry Environment
- **SENTRY_ENVIRONMENT**: Specifies the environment name for your Sentry events (e.g., "production", "staging", "development")
- Helps organize and filter errors by deployment environment in Sentry dashboard
- Example: `os.environ["SENTRY_ENVIRONMENT"] = "staging"`
- If not set, Sentry will use 'production' as the default environment
## Redacting Messages, Response Content from Sentry Logging
Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to sentry, but request metadata will still be logged.

266
docs/my-website/docs/ocr.md Normal file
View file

@ -0,0 +1,266 @@
# /ocr
| Feature | Supported |
|---------|-----------|
| Cost Tracking | ✅ |
| Logging | ✅ (Basic Logging not supported) |
| Load Balancing | ✅ |
| Supported Providers | `mistral`, `azure_ai`, `vertex_ai` |
:::tip
LiteLLM follows the [Mistral API request/response for the OCR API](https://docs.mistral.ai/capabilities/vision/#optical-character-recognition-ocr)
:::
## **LiteLLM Python SDK Usage**
### Quick Start
```python
from litellm import ocr
import os
os.environ["MISTRAL_API_KEY"] = "sk-.."
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "document_url",
"document_url": "https://arxiv.org/pdf/2201.04234"
}
)
# Access extracted text
for page in response.pages:
print(f"Page {page.index}:")
print(page.markdown)
```
### Async Usage
```python
from litellm import aocr
import os, asyncio
os.environ["MISTRAL_API_KEY"] = "sk-.."
async def test_async_ocr():
response = await aocr(
model="mistral/mistral-ocr-latest",
document={
"type": "document_url",
"document_url": "https://arxiv.org/pdf/2201.04234"
}
)
# Access extracted text
for page in response.pages:
print(f"Page {page.index}:")
print(page.markdown)
asyncio.run(test_async_ocr())
```
### Using Base64 Encoded Documents
```python
import base64
from litellm import ocr
# Encode PDF to base64
with open("document.pdf", "rb") as f:
base64_pdf = base64.b64encode(f.read()).decode('utf-8')
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "document_url",
"document_url": f"data:application/pdf;base64,{base64_pdf}"
}
)
```
### Optional Parameters
```python
response = ocr(
model="mistral/mistral-ocr-latest",
document={
"type": "document_url",
"document_url": "https://example.com/doc.pdf"
},
# Optional Mistral parameters
pages=[0, 1, 2], # Only process specific pages
include_image_base64=True, # Include extracted images
image_limit=10, # Max images to return
image_min_size=100 # Min image size to include
)
```
## **LiteLLM Proxy Usage**
LiteLLM provides a Mistral API compatible `/ocr` endpoint for OCR calls.
**Setup**
Add this to your litellm proxy config.yaml
```yaml
model_list:
- model_name: mistral-ocr
litellm_params:
model: mistral/mistral-ocr-latest
api_key: os.environ/MISTRAL_API_KEY
```
Start litellm
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
Test request
```bash
curl http://0.0.0.0:4000/v1/ocr \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "mistral-ocr",
"document": {
"type": "document_url",
"document_url": "https://arxiv.org/pdf/2201.04234"
}
}'
```
## **Request/Response Format**
:::info
LiteLLM follows the **Mistral OCR API specification**.
See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilities/vision/#optical-character-recognition-ocr) for complete details.
:::
### Example Request
```python
{
"model": "mistral/mistral-ocr-latest",
"document": {
"type": "document_url",
"document_url": "https://arxiv.org/pdf/2201.04234"
},
"pages": [0, 1, 2], # Optional: specific pages to process
"include_image_base64": True, # Optional: include extracted images
"image_limit": 10, # Optional: max images to return
"image_min_size": 100 # Optional: min image size in pixels
}
```
### Request Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) |
| `document` | object | Yes | Document to process. Must contain `type` and URL field |
| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images |
| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) |
| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) |
| `pages` | array | No | List of specific page indices to process (0-indexed) |
| `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings |
| `image_limit` | integer | No | Maximum number of images to return |
| `image_min_size` | integer | No | Minimum size (in pixels) for images to include |
#### Document Format Examples
**For PDFs and documents:**
```json
{
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
```
**For images:**
```json
{
"type": "image_url",
"image_url": "https://example.com/image.png"
}
```
**For base64-encoded content:**
```json
{
"type": "document_url",
"document_url": "data:application/pdf;base64,JVBERi0xLjQKJ..."
}
```
### Response Format
The response follows Mistral's OCR format with the following structure:
```json
{
"pages": [
{
"index": 0,
"markdown": "# Document Title\n\nExtracted text content...",
"dimensions": {
"dpi": 200,
"height": 2200,
"width": 1700
},
"images": [
{
"image_base64": "base64string...",
"bbox": {
"x": 100,
"y": 200,
"width": 300,
"height": 400
}
}
]
}
],
"model": "mistral-ocr-2505-completion",
"usage_info": {
"pages_processed": 29,
"doc_size_bytes": 3002783
},
"document_annotation": null,
"object": "ocr"
}
```
#### Response Fields
| Field | Type | Description |
|-------|------|-------------|
| `pages` | array | List of processed pages with extracted content |
| `pages[].index` | integer | Page number (0-indexed) |
| `pages[].markdown` | string | Extracted text in Markdown format |
| `pages[].dimensions` | object | Page dimensions (dpi, height, width in pixels) |
| `pages[].images` | array | Extracted images from the page (if `include_image_base64=true`) |
| `model` | string | The model used for OCR processing |
| `usage_info` | object | Processing statistics (pages processed, document size) |
| `document_annotation` | object | Optional document-level annotations |
| `object` | string | Always `"ocr"` for OCR responses |
## **Supported Providers**
| Provider | Link to Usage |
|-------------|--------------------|
| Mistral AI | [Usage](#quick-start) |
| Azure AI | [Usage](../docs/providers/azure_ocr) |
| Vertex AI | [Usage](../docs/providers/vertex_ocr) |

2
docs/my-website/docs/pass_through/anthropic_completion.md
View file

@ -1,7 +1,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Anthropic SDK
# Anthropic Passthrough
Pass-through endpoints for Anthropic - call provider-specific endpoint, in native format (no translation).

392
docs/my-website/docs/pass_through/bedrock.md
View file

@ -5,24 +5,55 @@ Pass-through endpoints for Bedrock - call provider-specific endpoint, in native
| Feature | Supported | Notes |
|-------|-------|-------|
| Cost Tracking | ✅ | For `/invoke` and `/converse` endpoints |
| Logging | ✅ | works across all integrations |
| Load Balancing | ✅ | You can load balance `/invoke`, `/converse` routes across multiple deployments| Logging | ✅ | works across all integrations |
| End-user Tracking | ❌ | [Tell us if you need this](https://github.com/BerriAI/litellm/issues/new) |
| Streaming | ✅ | |
Just replace `https://bedrock-runtime.{aws_region_name}.amazonaws.com` with `LITELLM_PROXY_BASE_URL/bedrock` 🚀
#### **Example Usage**
```bash
curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \
-H 'Authorization: Bearer anything' \
## Overview
LiteLLM supports two ways to call Bedrock endpoints:
### 1. **Using config.yaml** (Recommended for model endpoints)
Define your Bedrock models in `config.yaml` and reference them by name. The proxy handles authentication and routing.
**Use for**: `/converse`, `/converse-stream`, `/invoke`, `/invoke-with-response-stream`
```yaml showLineNumbers
model_list:
- model_name: my-bedrock-model
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-west-2
custom_llm_provider: bedrock
```
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"messages": [
{"role": "user",
"content": [{"text": "Hello"}]
}
]
}'
-d '{"messages": [{"role": "user", "content": [{"text": "Hello"}]}]}'
```
### 2. **Direct passthrough** (For non-model endpoints)
Set AWS credentials via environment variables and call Bedrock endpoints directly.
**Use for**: Guardrails, Knowledge Bases, Agents, and other non-model endpoints
```bash showLineNumbers
export AWS_ACCESS_KEY_ID=""
export AWS_SECRET_ACCESS_KEY=""
export AWS_REGION_NAME="us-west-2"
```
```bash showLineNumbers
curl "http://0.0.0.0:4000/bedrock/guardrail/my-guardrail-id/version/1/apply" \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{"contents": [{"text": {"text": "Hello"}}], "source": "INPUT"}'
```
Supports **ALL** Bedrock Endpoints (including streaming).
@ -33,39 +64,235 @@ Supports **ALL** Bedrock Endpoints (including streaming).
Let's call the Bedrock [`/converse` endpoint](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
1. Add AWS Keys to your environment
1. Create a `config.yaml` file with your Bedrock model
```bash
```yaml showLineNumbers
model_list:
- model_name: my-bedrock-model
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-west-2
custom_llm_provider: bedrock
```
Set your AWS credentials:
```bash showLineNumbers
export AWS_ACCESS_KEY_ID="" # Access key
export AWS_SECRET_ACCESS_KEY="" # Secret access key
export AWS_REGION_NAME="" # us-east-1, us-east-2, us-west-1, us-west-2
```
2. Start LiteLLM Proxy
```bash
litellm
```bash showLineNumbers
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
Let's call the Bedrock converse endpoint
Let's call the Bedrock converse endpoint using the model name from config:
```bash
curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \
-H 'Authorization: Bearer anything' \
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"messages": [
{"role": "user",
"content": [{"text": "Hello"}]
{
"role": "user",
"content": [{"text": "Hello, how are you?"}]
}
],
"inferenceConfig": {
"maxTokens": 100
}
]
}'
```
## Setup with config.yaml
Use config.yaml to define Bedrock models and use them via passthrough endpoints.
### 1. Define models in config.yaml
```yaml showLineNumbers
model_list:
- model_name: my-claude-model
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-west-2
custom_llm_provider: bedrock
- model_name: my-cohere-model
litellm_params:
model: bedrock/cohere.command-r-v1:0
aws_region_name: us-east-1
custom_llm_provider: bedrock
```
### 2. Start proxy with config
```bash showLineNumbers
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Call Bedrock Converse endpoint
Use the `model_name` from config in the URL path:
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/converse' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"messages": [
{
"role": "user",
"content": [{"text": "Hello, how are you?"}]
}
],
"inferenceConfig": {
"temperature": 0.5,
"maxTokens": 100
}
}'
```
### 4. Call Bedrock Converse Stream endpoint
For streaming responses, use the `/converse-stream` endpoint:
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/converse-stream' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"messages": [
{
"role": "user",
"content": [{"text": "Tell me a short story"}]
}
],
"inferenceConfig": {
"temperature": 0.7,
"maxTokens": 200
}
}'
```
### Supported Bedrock Endpoints with config.yaml
When using models from config.yaml, you can call any Bedrock endpoint:
| Endpoint | Description | Example |
|----------|-------------|---------|
| `/model/{model_name}/converse` | Converse API | `http://0.0.0.0:4000/bedrock/model/my-claude-model/converse` |
| `/model/{model_name}/converse-stream` | Streaming Converse | `http://0.0.0.0:4000/bedrock/model/my-claude-model/converse-stream` |
| `/model/{model_name}/invoke` | Legacy Invoke API | `http://0.0.0.0:4000/bedrock/model/my-claude-model/invoke` |
| `/model/{model_name}/invoke-with-response-stream` | Legacy Streaming | `http://0.0.0.0:4000/bedrock/model/my-claude-model/invoke-with-response-stream` |
The proxy automatically resolves the `model_name` to the actual Bedrock model ID and region configured in your `config.yaml`.
### Load Balancing Across Multiple Deployments
Define multiple Bedrock deployments with the same `model_name` to enable automatic load balancing.
#### 1. Define multiple deployments in config.yaml
```yaml showLineNumbers
model_list:
# First deployment - us-west-2
- model_name: my-claude-model
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-west-2
custom_llm_provider: bedrock
# Second deployment - us-east-1 (load balanced)
- model_name: my-claude-model
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-east-1
custom_llm_provider: bedrock
```
#### 2. Start proxy with config
```bash showLineNumbers
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### 3. Call the endpoint - requests are automatically load balanced
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/invoke' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"max_tokens": 100,
"messages": [
{
"role": "user",
"content": "Hello, how are you?"
}
],
"anthropic_version": "bedrock-2023-05-31"
}'
```
The proxy will automatically distribute requests across both `us-west-2` and `us-east-1` deployments. This works for all Bedrock endpoints: `/invoke`, `/invoke-with-response-stream`, `/converse`, and `/converse-stream`.
#### Using boto3 SDK with load balancing
You can also call the load-balanced endpoint using the boto3 SDK:
```python showLineNumbers
import boto3
import json
import os
# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy)
os.environ['AWS_ACCESS_KEY_ID'] = 'dummy'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'dummy'
os.environ['AWS_BEARER_TOKEN_BEDROCK'] = "sk-1234" # your litellm proxy api key
# Point boto3 to the LiteLLM proxy
bedrock_runtime = boto3.client(
service_name='bedrock-runtime',
region_name='us-west-2',
endpoint_url='http://0.0.0.0:4000/bedrock'
)
# Call the load-balanced model
response = bedrock_runtime.invoke_model(
modelId='my-claude-model', # Your model_name from config.yaml
contentType='application/json',
accept='application/json',
body=json.dumps({
"max_tokens": 100,
"messages": [
{
"role": "user",
"content": "Hello, how are you?"
}
],
"anthropic_version": "bedrock-2023-05-31"
})
)
# Parse response
response_body = json.loads(response['body'].read())
print(response_body['content'][0]['text'])
```
The proxy will automatically load balance your boto3 requests across all configured deployments.
## Examples
@ -84,7 +311,7 @@ Key Changes:
#### LiteLLM Proxy Call
```bash
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \
-H 'Authorization: Bearer sk-anything' \
-H 'Content-Type: application/json' \
@ -99,7 +326,7 @@ curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse'
#### Direct Bedrock API Call
```bash
```bash showLineNumbers
curl -X POST 'https://bedrock-runtime.us-west-2.amazonaws.com/model/cohere.command-r-v1:0/converse' \
-H 'Authorization: AWS4-HMAC-SHA256..' \
-H 'Content-Type: application/json' \
@ -114,9 +341,25 @@ curl -X POST 'https://bedrock-runtime.us-west-2.amazonaws.com/model/cohere.comma
### **Example 2: Apply Guardrail**
**Setup**: Set AWS credentials for direct passthrough
```bash showLineNumbers
export AWS_ACCESS_KEY_ID="your-access-key"
export AWS_SECRET_ACCESS_KEY="your-secret-key"
export AWS_REGION_NAME="us-west-2"
```
Start proxy:
```bash showLineNumbers
litellm
# RUNNING on http://0.0.0.0:4000
```
#### LiteLLM Proxy Call
```bash
```bash showLineNumbers
curl "http://0.0.0.0:4000/bedrock/guardrail/guardrailIdentifier/version/guardrailVersion/apply" \
-H 'Authorization: Bearer sk-anything' \
-H 'Content-Type: application/json' \
@ -129,7 +372,7 @@ curl "http://0.0.0.0:4000/bedrock/guardrail/guardrailIdentifier/version/guardrai
#### Direct Bedrock API Call
```bash
```bash showLineNumbers
curl "https://bedrock-runtime.us-west-2.amazonaws.com/guardrail/guardrailIdentifier/version/guardrailVersion/apply" \
-H 'Authorization: AWS4-HMAC-SHA256..' \
-H 'Content-Type: application/json' \
@ -142,7 +385,25 @@ curl "https://bedrock-runtime.us-west-2.amazonaws.com/guardrail/guardrailIdentif
### **Example 3: Query Knowledge Base**
```bash
**Setup**: Set AWS credentials for direct passthrough
```bash showLineNumbers
export AWS_ACCESS_KEY_ID="your-access-key"
export AWS_SECRET_ACCESS_KEY="your-secret-key"
export AWS_REGION_NAME="us-west-2"
```
Start proxy:
```bash showLineNumbers
litellm
# RUNNING on http://0.0.0.0:4000
```
#### LiteLLM Proxy Call
```bash showLineNumbers
curl -X POST "http://0.0.0.0:4000/bedrock/knowledgebases/{knowledgeBaseId}/retrieve" \
-H 'Authorization: Bearer sk-anything' \
-H 'Content-Type: application/json' \
@ -163,7 +424,7 @@ curl -X POST "http://0.0.0.0:4000/bedrock/knowledgebases/{knowledgeBaseId}/retri
#### Direct Bedrock API Call
```bash
```bash showLineNumbers
curl -X POST "https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases/{knowledgeBaseId}/retrieve" \
-H 'Authorization: AWS4-HMAC-SHA256..' \
-H 'Content-Type: application/json' \
@ -194,7 +455,7 @@ Use this, to avoid giving developers the raw AWS Keys, but still letting them us
1. Setup environment
```bash
```bash showLineNumbers
export DATABASE_URL=""
export LITELLM_MASTER_KEY=""
export AWS_ACCESS_KEY_ID="" # Access key
@ -202,7 +463,7 @@ export AWS_SECRET_ACCESS_KEY="" # Secret access key
export AWS_REGION_NAME="" # us-east-1, us-east-2, us-west-1, us-west-2
```
```bash
```bash showLineNumbers
litellm
# RUNNING on http://0.0.0.0:4000
@ -210,7 +471,7 @@ litellm
2. Generate virtual key
```bash
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
@ -219,7 +480,7 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \
Expected Response
```bash
```bash showLineNumbers
{
...
"key": "sk-1234ewknldferwedojwojw"
@ -229,7 +490,7 @@ Expected Response
3. Test it!
```bash
```bash showLineNumbers
curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \
-H 'Authorization: Bearer sk-1234ewknldferwedojwojw' \
-H 'Content-Type: application/json' \
@ -246,46 +507,46 @@ curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse'
Call Bedrock Agents via LiteLLM proxy
```python
**Setup**: Set AWS credentials on your LiteLLM proxy server
```bash showLineNumbers
export AWS_ACCESS_KEY_ID="your-access-key"
export AWS_SECRET_ACCESS_KEY="your-secret-key"
export AWS_REGION_NAME="us-west-2"
```
Start proxy:
```bash showLineNumbers
litellm
# RUNNING on http://0.0.0.0:4000
```
**Usage from Python**:
```python showLineNumbers
import os
import boto3
from botocore.config import Config
# # Define your proxy endpoint
proxy_endpoint = "http://0.0.0.0:4000/bedrock" # 👈 your proxy base url
# # Create a Config object with the proxy
# Custom headers
custom_headers = {
'litellm_user_api_key': 'Bearer sk-1234', # 👈 your proxy api key
}
os.environ["AWS_ACCESS_KEY_ID"] = "my-fake-key-id"
os.environ["AWS_SECRET_ACCESS_KEY"] = "my-fake-access-key"
import boto3
# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy)
os.environ["AWS_ACCESS_KEY_ID"] = "dummy"
os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy"
os.environ["AWS_BEARER_TOKEN_BEDROCK"] = "sk-1234" # your litellm proxy api key
# Create the client
runtime_client = boto3.client(
service_name="bedrock-agent-runtime",
region_name="us-west-2",
endpoint_url=proxy_endpoint
endpoint_url="http://0.0.0.0:4000/bedrock"
)
# Custom header injection
def inject_custom_headers(request, **kwargs):
request.headers.update(custom_headers)
# Attach the event to inject custom headers before the request is sent
runtime_client.meta.events.register('before-send.*.*', inject_custom_headers)
response = runtime_client.invoke_agent(
agentId="L1RT58GYRW",
agentAliasId="MFPSBCXYTW",
sessionId="12345",
inputText="Who do you know?"
)
agentId="L1RT58GYRW",
agentAliasId="MFPSBCXYTW",
sessionId="12345",
inputText="Who do you know?"
)
completion = ""
@ -294,5 +555,4 @@ for event in response.get("completion"):
completion += chunk["bytes"].decode()
print(completion)
```

3
docs/my-website/docs/pass_through/openai_passthrough.md
View file

@ -19,6 +19,9 @@ Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai`
## Usage Examples
Requirements:
Set `OPENAI_API_KEY` in your environment variables.
### Assistants API
#### Create OpenAI Client

4
docs/my-website/docs/pass_through/vertex_ai.md
View file

@ -18,8 +18,8 @@ Pass-through endpoints for Vertex AI - call provider-specific endpoint, in nativ
LiteLLM supports 3 vertex ai passthrough routes:
1. `/vertex_ai` → routes to `https://{vertex_location}-aiplatform.googleapis.com/`
2. `/vertex_ai/discovery` → routes to [`https://discoveryengine.googleapis.com`](https://discoveryengine.googleapis.com/)
3. `/vertex_ai/live` → upgrades to the Vertex AI Live API WebSocket (`google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent`)
2. `/vertex_ai/discovery` → routes to [`https://discoveryengine.googleapis.com`](https://discoveryengine.googleapis.com/) - [See Search Datastores Guide](./vertex_ai_search_datastores.md)
3. `/vertex_ai/live` → upgrades to the Vertex AI Live API WebSocket (`google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent`) - [See Live WebSocket Guide](./vertex_ai_live_websocket.md)
## How to use

139
docs/my-website/docs/pass_through/vertex_ai_search_datastores.md Normal file
View file

@ -0,0 +1,139 @@
# Vertex AI Search Datastores
Call Vertex AI Discovery Engine Search API through LiteLLM.
Provider Doc: https://cloud.google.com/generative-ai-app-builder/docs/reference/rest/v1/projects.locations.dataStores.servingConfigs/search
## What you get
- Reference datastores by ID. LiteLLM finds the credentials.
- No project/location in every request.
- Configure credentials once, use everywhere.
- Cost tracking works automatically.
## Quick Start
**Step 1. Set credentials**
```bash
export DEFAULT_VERTEXAI_PROJECT="your-project-id"
export DEFAULT_VERTEXAI_LOCATION="us-central1"
export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="/path/to/credentials.json"
```
**Step 2. Start proxy**
```bash
litellm
```
**Step 3. Search your datastore**
```bash
curl -X POST \
"http://localhost:4000/vertex_ai/discovery/v1/projects/my-project/locations/global/collections/default_collection/dataStores/my-datastore/servingConfigs/default_config:search" \
-H "Content-Type: application/json" \
-H "x-litellm-api-key: Bearer sk-1234" \
-d '{
"query": "How do I authenticate?",
"pageSize": 10
}'
```
## Managed Vector Stores (Recommended)
Register your datastore once. Reference it by ID.
**In config.yaml:**
```yaml
vector_store_registry:
- vector_store_name: "vertex-ai-litellm-website-knowledgebase"
litellm_params:
vector_store_id: "my-datastore"
custom_llm_provider: "vertex_ai/search_api"
vertex_app_id: "test-litellm-app_1761094730750"
vertex_project: "test-vector-store-db"
vertex_location: "global"
vector_store_description: "Vertex AI vector store for the Litellm website knowledgebase"
vector_store_metadata:
source: "https://www.litellm.com/docs"
```
**How it works:**
LiteLLM sees `dataStores/my-datastore` in your URL. It looks up the vector store. Uses the right project and credentials automatically.
## Endpoint
`{PROXY_BASE_URL}/vertex_ai/discovery/{endpoint:path}`
Routes to `https://discoveryengine.googleapis.com`
## Examples
### Basic Search
```bash
curl -X POST \
"http://localhost:4000/vertex_ai/discovery/v1/projects/my-project/locations/global/collections/default_collection/dataStores/my-datastore/servingConfigs/default_config:search" \
-H "Content-Type: application/json" \
-H "x-litellm-api-key: Bearer sk-1234" \
-d '{
"query": "pricing",
"pageSize": 10
}'
```
### Search with Filters
```bash
curl -X POST \
"http://localhost:4000/vertex_ai/discovery/v1/projects/my-project/locations/global/collections/default_collection/dataStores/my-datastore/servingConfigs/default_config:search" \
-H "Content-Type: application/json" \
-H "x-litellm-api-key: Bearer sk-1234" \
-d '{
"query": "tutorials",
"pageSize": 20,
"filter": "category = \"beginner\"",
"spellCorrectionSpec": {"mode": "AUTO"}
}'
```
### Python
```python
import requests
url = "http://localhost:4000/vertex_ai/discovery/v1/projects/my-project/locations/global/collections/default_collection/dataStores/my-datastore/servingConfigs/default_config:search"
response = requests.post(url,
headers={
"Content-Type": "application/json",
"x-litellm-api-key": "Bearer sk-1234"
},
json={"query": "pricing", "pageSize": 10}
)
for result in response.json().get("results", []):
data = result["document"]["derivedStructData"]
print(f"{data['title']}: {data['link']}")
```
### Use with Chat Completion
```bash
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "claude-3-5-sonnet",
"messages": [{"role": "user", "content": "What is litellm?"}],
"tools": [
{
"type": "file_search",
"vector_store_ids": ["my-datastore"]
}
]
}'
```

7
docs/my-website/docs/projects/Softgen.md Normal file
View file

@ -0,0 +1,7 @@
# Softgen
`Softgen` is an AI-powered platform that builds full-stack web apps from your plain instructions.
LiteLLM helps `Softgen` users to choose and use different LLMs.
- [Softgen](https://softgen.ai)
- [Academy](hhttps://academy.softgen.ai)

68
docs/my-website/docs/providers/anthropic.md
View file

@ -953,7 +953,7 @@ except Exception as e:
s/o @[Shekhar Patnaik](https://www.linkedin.com/in/patnaikshekhar) for requesting this!
### Anthropic Hosted Tools (Computer, Text Editor, Web Search)
### Anthropic Hosted Tools (Computer, Text Editor, Web Search, Memory)
<Tabs>
@ -1183,6 +1183,72 @@ curl http://0.0.0.0:4000/v1/chat/completions \
</Tabs>
</TabItem>
<TabItem value="memory" label="Memory">
:::info
The Anthropic Memory tool is currently in beta.
:::
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
tools = [{
"type": "memory_20250818",
"name": "memory"
}]
model = "claude-sonnet-4-5-20250929"
messages = [{"role": "user", "content": "Please remember that my favorite color is blue."}]
response = completion(
model=model,
messages=messages,
tools=tools,
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
1. Setup config.yaml
```yaml
model_list:
- model_name: claude-memory-model
litellm_params:
model: anthropic/claude-sonnet-4-5-20250929
api_key: os.environ/ANTHROPIC_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "claude-memory-model",
"messages": [{"role": "user", "content": "Please remember that my favorite color is blue."}],
"tools": [{"type": "memory_20250818", "name": "memory"}]
}'
```
</TabItem>
</Tabs>
</TabItem>
</Tabs>

45
docs/my-website/docs/providers/azure/azure.md
View file

@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem';
|-------|-------|
| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series |
| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models) |
| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview)
## API Keys, Params
@ -538,39 +538,6 @@ response = litellm.completion(
print(response)
```
## Azure Text to Speech (tts)
**LiteLLM PROXY**
```yaml
- model_name: azure/tts-1
litellm_params:
model: azure/tts-1
api_base: "os.environ/AZURE_API_BASE_TTS"
api_key: "os.environ/AZURE_API_KEY_TTS"
api_version: "os.environ/AZURE_API_VERSION"
```
**LiteLLM SDK**
```python
from litellm import completion
## set ENV variables
os.environ["AZURE_API_KEY"] = ""
os.environ["AZURE_API_BASE"] = ""
os.environ["AZURE_API_VERSION"] = ""
# azure call
speech_file_path = Path(__file__).parent / "speech.mp3"
response = speech(
model="azure/<your-deployment-name",
voice="alloy",
input="the quick brown fox jumped over the lazy dogs",
)
response.stream_to_file(speech_file_path)
```
## **Authentication**
@ -867,7 +834,7 @@ client = OpenAI(
batch_input_file = client.files.create(
file=open("mydata.jsonl", "rb"),
purpose="batch",
extra_body={"custom_llm_provider": "azure"}
extra_headers={"custom-llm-provider": "azure"}
)
file_id = batch_input_file.id
```
@ -903,7 +870,7 @@ batch = client.batches.create( # re use client from above
endpoint="/v1/chat/completions",
completion_window="24h",
metadata={"description": "My batch job"},
extra_body={"custom_llm_provider": "azure"}
extra_headers={"custom-llm-provider": "azure"}
)
```
@ -931,7 +898,7 @@ curl http://localhost:4000/v1/batches \
```python
retrieved_batch = client.batches.retrieve(
batch.id,
extra_query={"custom_llm_provider": "azure"}
extra_headers={"custom-llm-provider": "azure"}
)
```
@ -955,7 +922,7 @@ curl http://localhost:4000/v1/batches/batch_abc123 \
```python
cancelled_batch = client.batches.cancel(
batch.id,
extra_body={"custom_llm_provider": "azure"}
extra_headers={"custom-llm-provider": "azure"}
)
```
@ -978,7 +945,7 @@ curl http://localhost:4000/v1/batches/batch_abc123/cancel \
<TabItem value="sdk" label="OpenAI Python SDK">
```python
client.batches.list(extra_query={"custom_llm_provider": "azure"})
client.batches.list(extra_headers={"custom-llm-provider": "azure"})
```
</TabItem>

75
docs/my-website/docs/providers/azure/azure_speech.md Normal file
View file

@ -0,0 +1,75 @@
# Azure Text to Speech (tts)
## Overview
| Property | Details |
|-------|-------|
| Description | Convert text to natural-sounding speech using Azure OpenAI's Text to Speech models |
| Provider Route on LiteLLM | `azure/` |
| Supported Operations | `/audio/speech` |
| Link to Provider Doc | [Azure OpenAI TTS ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/text-to-speech-quickstart)
## Quick Start
### **LiteLLM SDK**
```python showLineNumbers title="SDK Usage"
from litellm import speech
from pathlib import Path
import os
## set ENV variables
os.environ["AZURE_API_KEY"] = ""
os.environ["AZURE_API_BASE"] = ""
os.environ["AZURE_API_VERSION"] = ""
# azure call
speech_file_path = Path(__file__).parent / "speech.mp3"
response = speech(
model="azure/<your-deployment-name>",
voice="alloy",
input="the quick brown fox jumped over the lazy dogs",
)
response.stream_to_file(speech_file_path)
```
### **LiteLLM PROXY**
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
- model_name: azure/tts-1
litellm_params:
model: azure/tts-1
api_base: "os.environ/AZURE_API_BASE_TTS"
api_key: "os.environ/AZURE_API_KEY_TTS"
api_version: "os.environ/AZURE_API_VERSION"
```
## Available Voices
Azure OpenAI supports the following voices:
- `alloy` - Neutral and balanced
- `echo` - Warm and upbeat
- `fable` - Expressive and dramatic
- `onyx` - Deep and authoritative
- `nova` - Friendly and conversational
- `shimmer` - Bright and cheerful
## Supported Parameters
```python showLineNumbers title="All Parameters"
response = speech(
model="azure/<your-deployment-name>",
voice="alloy", # Required: Voice selection
input="text to convert", # Required: Input text
speed=1.0, # Optional: 0.25 to 4.0 (default: 1.0)
response_format="mp3" # Optional: mp3, opus, aac, flac, wav, pcm
)
```
## Supported Models
- `tts-1` - Standard quality, optimized for speed
- `tts-1-hd` - High definition, optimized for quality
Use your Azure deployment name: `azure/<your-deployment-name>`

282
docs/my-website/docs/providers/azure/videos.md Normal file
View file

@ -0,0 +1,282 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Azure Video Generation
LiteLLM supports Azure OpenAI's video generation models including Sora with full end-to-end integration.
| Property | Details |
|-------|-------|
| Description | Azure OpenAI's video generation models including Sora-2 |
| Provider Route on LiteLLM | `azure/` |
| Supported Models | `sora-2` |
| Cost Tracking | ✅ Duration-based pricing ($0.10/second) |
| Logging Support | ✅ Full request/response logging |
| Guardrails Support | ✅ Content moderation and safety checks |
| Proxy Server Support | ✅ Full proxy integration with virtual keys |
| Spend Management | ✅ Budget tracking and rate limiting |
| Link to Provider Doc | [Azure OpenAI Video Generation ↗](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/video-generation) |
## Quick Start
### Required API Keys
```python
import os
os.environ["AZURE_OPENAI_API_KEY"] = "your-azure-api-key"
os.environ["AZURE_OPENAI_API_BASE"] = "https://your-resource.openai.azure.com/"
```
### Basic Usage
```python
from litellm import video_generation, video_status, video_content
import os
import time
os.environ["AZURE_OPENAI_API_KEY"] = "your-azure-api-key"
os.environ["AZURE_OPENAI_API_BASE"] = "https://your-resource.openai.azure.com/"
# Generate video
response = video_generation(
model="azure/sora-2",
prompt="A cat playing with a ball of yarn in a sunny garden",
seconds="8",
size="720x1280"
)
print(f"Video ID: {response.id}")
print(f"Initial Status: {response.status}")
# Check status until video is ready
while True:
status_response = video_status(
video_id=response.id
)
print(f"Current Status: {status_response.status}")
if status_response.status == "completed":
break
elif status_response.status == "failed":
print("Video generation failed")
break
time.sleep(10) # Wait 10 seconds before checking again
# Download video content when ready
video_bytes = video_content(
video_id=response.id
)
# Save to file
with open("generated_video.mp4", "wb") as f:
f.write(video_bytes)
```
## Usage - LiteLLM Proxy Server
Here's how to call Azure video generation models with the LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export AZURE_OPENAI_API_KEY="your-azure-api-key"
export AZURE_OPENAI_API_BASE="https://your-resource.openai.azure.com/"
```
### 2. Start the proxy
<Tabs>
<TabItem value="config" label="config.yaml">
```yaml
model_list:
- model_name: azure-sora-2
litellm_params:
model: azure/sora-2
api_key: os.environ/AZURE_OPENAI_API_KEY
api_base: os.environ/AZURE_OPENAI_API_BASE
```
</TabItem>
<TabItem value="cli" label="CLI">
```bash
$ litellm --model azure/sora-2
# Server running on http://0.0.0.0:4000
```
</TabItem>
</Tabs>
### 3. Test it
<Tabs>
<TabItem value="Curl" label="Curl Request">
```shell
curl --location 'http://0.0.0.0:4000/videos/generations' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "azure-sora-2",
"prompt": "A cat playing with a ball of yarn in a sunny garden",
"seconds": "8",
"size": "720x1280"
}'
```
</TabItem>
<TabItem value="openai" label="OpenAI v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
# request sent to model set on litellm proxy, `litellm --model`
response = client.videos.create(
model="azure-sora-2",
prompt="A cat playing with a ball of yarn in a sunny garden",
seconds=8,
size="720x1280"
)
print(response)
```
</TabItem>
</Tabs>
## Supported Models
| Model Name |
|------------|
| sora-2 |
|sora-2-pro |
|sora-2-pro-high-res|
## Logging & Observability
### Request/Response Logging
All video generation requests are automatically logged with:
- **Request details**: prompt, model, duration, size
- **Response details**: video ID, status, creation time
- **Cost tracking**: duration-based pricing calculation
- **Performance metrics**: request latency, processing time
### Logging Providers
Video generation works with all LiteLLM logging providers:
- **Datadog**: Real-time monitoring and alerting
- **Helicone**: Request tracing and debugging
- **LangSmith**: LangChain integration and tracing
- **Custom webhooks**: Send logs to your own endpoints
**Example: Enable Datadog logging**
```yaml
general_settings:
alerting: ["datadog"]
datadog_api_key: os.environ/DATADOG_API_KEY
```
## Video Generation Parameters
- `prompt` (required): Text description of the desired video
- `model` (optional): Model to use, defaults to "azure/sora-2"
- `seconds` (optional): Video duration in seconds (e.g., "8", "16")
- `size` (optional): Video dimensions (e.g., "720x1280", "1280x720")
- `input_reference` (optional): Reference image for video editing
- `user` (optional): User identifier for tracking
## Video Content Retrieval
```python
# Download video content
video_bytes = video_content(
video_id="video_1234567890"
)
# Save to file
with open("video.mp4", "wb") as f:
f.write(video_bytes)
```
## Complete Workflow
```python
import litellm
import time
def generate_and_download_video(prompt):
# Step 1: Generate video
response = litellm.video_generation(
prompt=prompt,
model="azure/sora-2",
seconds="8",
size="720x1280"
)
video_id = response.id
print(f"Video ID: {video_id}")
# Step 2: Wait for processing (in practice, poll status)
time.sleep(30)
# Step 3: Download video
video_bytes = litellm.video_content(
video_id=video_id
)
# Step 4: Save to file
with open(f"video_{video_id}.mp4", "wb") as f:
f.write(video_bytes)
return f"video_{video_id}.mp4"
# Usage
video_file = generate_and_download_video(
"A cat playing with a ball of yarn in a sunny garden"
)
```
## Video Remix (Video Editing)
```python
# Video editing with reference image
response = litellm.video_remix(
video_id="video_456",
prompt="Make the cat jump higher",
input_reference=open("path/to/image.jpg", "rb"), # Reference image as file object
seconds="8"
)
print(f"Video ID: {response.id}")
```
## Error Handling
```python
from litellm.exceptions import BadRequestError, AuthenticationError
try:
response = video_generation(
prompt="A cat playing with a ball of yarn",
model="azure/sora-2"
)
except AuthenticationError as e:
print(f"Authentication failed: {e}")
except BadRequestError as e:
print(f"Bad request: {e}")
```

391
docs/my-website/docs/providers/azure_ai/azure_ai_vector_stores_passthrough.md Normal file
View file

@ -0,0 +1,391 @@
# Azure AI Search - Vector Store (Passthrough API)
Use this to allow developers to **create** and **search** vector stores using the Azure AI Search API in the **native** Azure AI Search API format, without giving them the Azure AI credentials.
This is for the proxy only.
## Admin Flow
### 1. Add the vector store to LiteLLM
```yaml
model_list:
- model_name: embedding-model
litellm_params:
model: openai/text-embedding-3-large
vector_store_registry:
- vector_store_name: "azure-ai-search"
litellm_params:
vector_store_id: "can-be-anything" # vector store id can be anything for the purpose of passthrough api
custom_llm_provider: "azure_ai"
api_key: os.environ/AZURE_SEARCH_API_KEY
api_base: https://azure-kb-search.search.windows.net
litellm_embedding_model: "azure/text-embedding-3-large"
litellm_embedding_config:
api_base: https://krris-mh44uf7y-eastus2.cognitiveservices.azure.com/
api_key: os.environ/AZURE_API_KEY
api_version: "2025-09-01"
general_settings:
database_url: "postgresql://user:password@host:port/database"
master_key: "sk-1234"
```
Add your vector store credentials to LiteLLM.
### 2. Start the proxy.
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Create a virtual index.
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/indexes' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"index_name": "dall-e-4",
"litellm_params": {
"vector_store_index": "real-index-name-2",
"vector_store_name": "azure-ai-search"
}
}'
```
This is a virtual index, which the developer can use to create and search vector stores.
### 4. Create a key with the vector store permissions.
```bash
curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"allowed_vector_store_indexes": [{"index_name": "dall-e-4", "index_permissions": ["write", "read"]}],
"models": ["embedding-model"]
}'
```
Give the key access to the virtual index and the embedding model.
**Expected response**
```json
{
"key": "sk-my-virtual-key"
}
```
## Developer Flow
### 1. Create a vector store with some documents.
Note: Use the '/azure_ai' endpoint for the passthrough api that uses the `azure_ai` provider in your `_new_secret_config.yaml` file.
```python
import requests
import json
# ----------------------------
# 🔐 CONFIGURATION
# ----------------------------
# Azure OpenAI (for embeddings)
AZURE_OPENAI_ENDPOINT = "http://0.0.0.0:4000"
AZURE_OPENAI_KEY = "sk-my-virtual-key"
EMBEDDING_DEPLOYMENT_NAME = "embedding-model"
# Azure AI Search
AZURE_AI_SEARCH_ENDPOINT = "http://0.0.0.0:4000/azure_ai" # IMPORTANT: Use the '/azure_ai' endpoint for the passthrough api to Azure
SEARCH_API_KEY = "sk-my-virtual-key"
INDEX_NAME = "dall-e-4"
# Vector dimensions (text-embedding-3-large uses 3072 dimensions)
VECTOR_DIMENSIONS = 3072
# Example docs (replace with your own)
documents = [
{"id": "1", "content": "Refunds must be requested within 30 days."},
{"id": "2", "content": "We offer 24/7 support for all enterprise customers."},
]
# ----------------------------
# 📋 STEP 0 — Create Index Schema
# ----------------------------
def delete_index_if_exists():
"""Delete the index if it exists"""
index_url = f"{AZURE_AI_SEARCH_ENDPOINT}/indexes/{INDEX_NAME}?api-version=2024-07-01"
headers = {"api-key": SEARCH_API_KEY}
response = requests.delete(index_url, headers=headers)
if response.status_code == 204:
print(f"🗑️ Deleted existing index '{INDEX_NAME}'")
return True
elif response.status_code == 404:
print(f"ℹ️ Index '{INDEX_NAME}' does not exist yet")
return False
else:
print(f"⚠️ Delete response: {response.status_code}")
print(f" Message: {response.text}")
return False
def create_index():
"""Create the Azure AI Search index with proper schema"""
index_url = f"{AZURE_AI_SEARCH_ENDPOINT}/indexes/{INDEX_NAME}?api-version=2024-07-01"
headers = {"Content-Type": "application/json", "api-key": SEARCH_API_KEY}
index_schema = {
"name": INDEX_NAME,
"fields": [
{"name": "id", "type": "Edm.String", "key": True, "filterable": True},
{
"name": "content",
"type": "Edm.String",
"searchable": True,
"filterable": False,
},
{
"name": "contentVector",
"type": "Collection(Edm.Single)",
"searchable": True,
"dimensions": VECTOR_DIMENSIONS,
"vectorSearchProfile": "my-vector-profile",
},
],
"vectorSearch": {
"algorithms": [
{
"name": "my-hnsw-algorithm",
"kind": "hnsw",
"hnswParameters": {
"metric": "cosine",
"m": 4,
"efConstruction": 400,
"efSearch": 500,
},
}
],
"profiles": [
{"name": "my-vector-profile", "algorithm": "my-hnsw-algorithm"}
],
},
}
# Create the index
response = requests.put(index_url, headers=headers, json=index_schema)
if response.status_code == 201:
print(f"✅ Index '{INDEX_NAME}' created successfully.")
return True
elif response.status_code == 204:
print(f"✅ Index '{INDEX_NAME}' updated successfully.")
return True
else:
print(f"❌ Failed to create index: {response.status_code}")
print(f" Message: {response.text}")
return False
# Delete and recreate the index with correct schema
print("🔄 Setting up Azure AI Search index...")
delete_index_if_exists()
if not create_index():
print("❌ Could not create index. Exiting.")
exit(1)
# ----------------------------
# 🧠 STEP 1 — Generate Embeddings
# ----------------------------
def get_embedding(text: str):
url = f"{AZURE_OPENAI_ENDPOINT}/openai/deployments/{EMBEDDING_DEPLOYMENT_NAME}/embeddings?api-version=2024-10-21"
headers = {"Content-Type": "application/json", "api-key": AZURE_OPENAI_KEY}
payload = {"input": text}
response = requests.post(url, headers=headers, json=payload)
if response.status_code != 200:
raise Exception(f"Embedding failed: {response.status_code}\n{response.text}")
return response.json()["data"][0]["embedding"]
# Generate embeddings for each document
for doc in documents:
doc["contentVector"] = get_embedding(doc["content"])
print(f"✅ Embedded doc {doc['id']} (vector length: {len(doc['contentVector'])})")
# ----------------------------
# 📤 STEP 2 — Upload to Azure AI Search
# ----------------------------
upload_url = f"{AZURE_AI_SEARCH_ENDPOINT}/indexes/{INDEX_NAME}/docs/index?api-version=2024-07-01"
headers = {"Content-Type": "application/json", "api-key": SEARCH_API_KEY}
payload = {
"value": [
{
"@search.action": "upload",
"id": doc["id"],
"content": doc["content"],
"contentVector": doc["contentVector"],
}
for doc in documents
]
}
response = requests.post(upload_url, headers=headers, data=json.dumps(payload))
# ----------------------------
# 🧾 RESULT
# ----------------------------
if response.status_code == 200:
print("✅ Documents uploaded successfully.")
else:
print(f"❌ Upload failed: {response.status_code}")
print(response.text)
```
### 2. Search the vector store.
```python
import requests
import json
# ----------------------------
# 🔐 CONFIGURATION
# ----------------------------
# Azure OpenAI (for embeddings)
AZURE_OPENAI_ENDPOINT = "http://0.0.0.0:4000"
AZURE_OPENAI_KEY = "sk-my-virtual-key"
EMBEDDING_DEPLOYMENT_NAME = "embedding-model"
# Azure AI Search
AZURE_AI_SEARCH_ENDPOINT = "http://0.0.0.0:4000/azure_ai"
SEARCH_API_KEY = "sk-my-virtual-key"
INDEX_NAME = "dall-e-4"
# ----------------------------
# 🧠 Generate Query Embedding
# ----------------------------
def get_embedding(text: str):
"""Generate embedding for the query text"""
url = f"{AZURE_OPENAI_ENDPOINT}/openai/deployments/{EMBEDDING_DEPLOYMENT_NAME}/embeddings?api-version=2024-10-21"
headers = {"Content-Type": "application/json", "api-key": AZURE_OPENAI_KEY}
payload = {"input": text}
response = requests.post(url, headers=headers, json=payload)
if response.status_code != 200:
raise Exception(f"Embedding failed: {response.status_code}\n{response.text}")
return response.json()["data"][0]["embedding"]
# ----------------------------
# 🔍 Vector Search Function
# ----------------------------
def search_knowledge_base(query: str, top_k: int = 3):
"""
Search the knowledge base using vector similarity
Args:
query: The search query string
top_k: Number of top results to return (default: 3)
Returns:
List of search results with content and scores
"""
print(f"🔍 Searching for: '{query}'")
# Step 1: Generate embedding for the query
print(" Generating query embedding...")
query_vector = get_embedding(query)
# Step 2: Perform vector search
search_url = f"{AZURE_AI_SEARCH_ENDPOINT}/indexes/{INDEX_NAME}/docs/search?api-version=2024-07-01"
headers = {"Content-Type": "application/json", "api-key": SEARCH_API_KEY}
# Build the search request with vector search
search_payload = {
"search": "*", # Get all documents
"vectorQueries": [
{
"vector": query_vector,
"fields": "contentVector",
"kind": "vector",
"k": top_k, # Number of nearest neighbors to return
}
],
"select": "id,content", # Fields to return
"top": top_k,
}
# Execute the search
response = requests.post(search_url, headers=headers, json=search_payload)
if response.status_code != 200:
raise Exception(f"Search failed: {response.status_code}\n{response.text}")
# Parse and return results
results = response.json()
return results.get("value", [])
# ----------------------------
# 📊 Display Results
# ----------------------------
def display_results(results):
"""Pretty print the search results"""
if not results:
print("\n❌ No results found.")
return
print(f"\n✅ Found {len(results)} results:\n")
print("=" * 80)
for i, result in enumerate(results, 1):
print(f"\n📄 Result #{i}")
print(f" ID: {result.get('id', 'N/A')}")
print(f" Score: {result.get('@search.score', 'N/A')}")
print(f" Content: {result.get('content', 'N/A')}")
print("-" * 80)
# ----------------------------
# 🎯 MAIN - Example Queries
# ----------------------------
if __name__ == "__main__":
# Example 1: Search for refund policy
print("\n" + "=" * 80)
print("EXAMPLE 1: Refund Policy Query")
print("=" * 80)
results = search_knowledge_base("How do I get a refund?", top_k=2)
display_results(results)
# Example 2: Search for customer support
print("\n\n" + "=" * 80)
print("EXAMPLE 2: Customer Support Query")
print("=" * 80)
results = search_knowledge_base("When can I contact support?", top_k=2)
display_results(results)
# Example 3: Custom query - uncomment to use
# print("\n\n" + "=" * 80)
# print("CUSTOM QUERY")
# print("=" * 80)
# custom_query = input("Enter your query: ")
# results = search_knowledge_base(custom_query, top_k=3)
# display_results(results)
```

374
docs/my-website/docs/providers/azure_ai_speech.md Normal file
View file

@ -0,0 +1,374 @@
# Azure AI Speech (Cognitive Services)
Azure AI Speech is Azure's Cognitive Services text-to-speech API, separate from Azure OpenAI. It provides high-quality neural voices with broader language support and advanced speech customization.
**When to use this vs Azure OpenAI TTS:**
- **Azure AI Speech** - More languages, neural voices, SSML support, speech customization
- **Azure OpenAI TTS** - OpenAI models, integrated with Azure OpenAI services
## Overview
| Property | Details |
|-------|-------|
| Description | Azure AI Speech is Azure's Cognitive Services text-to-speech API, separate from Azure OpenAI. It provides high-quality neural voices with broader language support and advanced speech customization. |
| Provider Route on LiteLLM | `azure/speech/` |
## Quick Start
**LiteLLM SDK**
```python showLineNumbers title="SDK Usage"
from litellm import speech
from pathlib import Path
import os
os.environ["AZURE_TTS_API_KEY"] = "your-cognitive-services-key"
speech_file_path = Path(__file__).parent / "speech.mp3"
response = speech(
model="azure/speech/azure-tts",
voice="alloy",
input="Hello, this is Azure AI Speech",
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
)
response.stream_to_file(speech_file_path)
```
**LiteLLM Proxy**
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
- model_name: azure-speech
litellm_params:
model: azure/speech/azure-tts
api_base: https://eastus.tts.speech.microsoft.com
api_key: os.environ/AZURE_TTS_API_KEY
```
## Setup
1. Create an Azure Cognitive Services resource in the [Azure Portal](https://portal.azure.com)
2. Get your API key from the resource
3. Note your region (e.g., `eastus`, `westus`, `westeurope`)
4. Use the regional endpoint: `https://{region}.tts.speech.microsoft.com`
## Cost Tracking (Pricing)
LiteLLM automatically tracks costs for Azure AI Speech based on the number of characters processed.
### Available Models
| Model | Voice Type | Cost per 1M Characters |
|-------|-----------|----------------------|
| `azure/speech/azure-tts` | Neural | $15 |
| `azure/speech/azure-tts-hd` | Neural HD | $30 |
### How Costs are Calculated
Azure AI Speech charges based on the number of characters in your input text. LiteLLM automatically:
- Counts the number of characters in your `input` parameter
- Calculates the cost based on the model pricing
- Returns the cost in the response object
```python showLineNumbers title="View Request Cost"
from litellm import speech
response = speech(
model="azure/speech/azure-tts",
voice="alloy",
input="Hello, this is a test message",
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
)
# Access the calculated cost
cost = response._hidden_params.get("response_cost")
print(f"Request cost: ${cost}")
```
### Verify Azure Pricing
To check the latest Azure AI Speech pricing:
1. Visit the [Azure Pricing Calculator](https://azure.microsoft.com/en-us/pricing/calculator/)
2. Set **Service** to "AI Services"
3. Set **API** to "Azure AI Speech"
4. Select **Text to Speech** and your region
5. View the current pricing per million characters
**Note:** Pricing may vary by region and Azure subscription type.
## Voice Mapping
LiteLLM automatically maps OpenAI voice names to Azure Neural voices:
| OpenAI Voice | Azure Neural Voice | Description |
|-------------|-------------------|-------------|
| `alloy` | en-US-JennyNeural | Neutral and balanced |
| `echo` | en-US-GuyNeural | Warm and upbeat |
| `fable` | en-GB-RyanNeural | Expressive and dramatic |
| `onyx` | en-US-DavisNeural | Deep and authoritative |
| `nova` | en-US-AmberNeural | Friendly and conversational |
| `shimmer` | en-US-AriaNeural | Bright and cheerful |
## Supported Parameters
```python showLineNumbers title="All Parameters"
response = speech(
model="azure/speech/azure-tts",
voice="alloy", # Required: Voice selection
input="text to convert", # Required: Input text
speed=1.0, # Optional: 0.25 to 4.0 (default: 1.0)
response_format="mp3", # Optional: mp3, opus, wav, pcm
api_base="https://eastus.tts.speech.microsoft.com",
api_key="your-key",
)
```
### Response Formats
| Format | Azure Output Format | Sample Rate |
|--------|-------------------|-------------|
| `mp3` | audio-24khz-48kbitrate-mono-mp3 | 24kHz |
| `opus` | ogg-48khz-16bit-mono-opus | 48kHz |
| `wav` | riff-24khz-16bit-mono-pcm | 24kHz |
| `pcm` | raw-24khz-16bit-mono-pcm | 24kHz |
## Sending Azure-Specific Params
Azure AI Speech supports advanced SSML features through optional parameters:
- `style`: Speaking style (e.g., "cheerful", "sad", "angry", "whispering")
- `styledegree`: Style intensity (0.01 to 2)
- `role`: Voice role (e.g., "Girl", "Boy", "SeniorFemale", "SeniorMale")
- `lang`: Language code for multilingual voices (e.g., "es-ES", "fr-FR", "hi-IN")
### **LiteLLM SDK**
#### Custom Azure Voice
```python showLineNumbers title="Custom Azure Voice"
from litellm import speech
response = speech(
model="azure/speech/azure-tts",
voice="en-US-AndrewNeural", # Use Azure voice directly
input="Hello, this is a test",
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
response_format="mp3"
)
response.stream_to_file("speech.mp3")
```
#### Speaking Style
```python showLineNumbers title="Speaking Style"
from litellm import speech
response = speech(
model="azure/speech/azure-tts",
voice="en-US-JennyNeural", # Must be a voice that supports styles
input="Who are you? What is chicken dinner?",
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
style="whispering", # Azure-specific: cheerful, sad, angry, whispering, etc.
)
response.stream_to_file("speech.mp3")
```
#### Style with Degree and Role
```python showLineNumbers title="Style with Degree and Role"
from litellm import speech
response = speech(
model="azure/speech/azure-tts",
voice="en-US-AriaNeural",
input="Good morning! How are you today?",
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
style="cheerful", # Azure-specific: Speaking style
styledegree="2", # Azure-specific: 0.01 to 2 (intensity)
role="SeniorFemale", # Azure-specific: Girl, Boy, SeniorFemale, etc.
)
response.stream_to_file("speech.mp3")
```
#### Language Override for Multilingual Voices
```python showLineNumbers title="Language Override"
from litellm import speech
response = speech(
model="azure/speech/azure-tts",
voice="en-US-AvaMultilingualNeural", # Multilingual voice
input="आप कौन हैं? चिकन डिनर क्या है?", # Hindi text
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
lang="hi-IN", # Azure-specific: Override language
)
response.stream_to_file("speech.mp3")
```
### **LiteLLM AI Gateway (CURL)**
First, ensure you have set up your proxy config as shown in the [LiteLLM Proxy setup](#quick-start) above.
**Using the model name from your config:**
```yaml
model_list:
- model_name: azure-speech # This is what you'll use in your API calls
litellm_params:
model: azure/speech/azure-tts
api_base: https://eastus.tts.speech.microsoft.com
api_key: os.environ/AZURE_TTS_API_KEY
```
#### Custom Azure Voice
```bash
curl http://0.0.0.0:4000/v1/audio/speech \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "azure-speech",
"voice": "en-US-AndrewNeural",
"input": "Hello, this is a test"
}' \
--output speech.mp3
```
#### Speaking Style
```bash
curl http://0.0.0.0:4000/v1/audio/speech \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "azure-speech",
"input": "Who are you? What is chicken dinner?",
"voice": "en-US-JennyNeural",
"style": "whispering"
}' \
--output speech.mp3
```
#### Style with Degree and Role
```bash
curl http://0.0.0.0:4000/v1/audio/speech \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "azure-speech",
"voice": "en-US-AriaNeural",
"input": "Good morning! How are you today?",
"style": "cheerful",
"styledegree": "2",
"role": "SeniorFemale"
}' \
--output speech.mp3
```
#### Language Override
```bash
curl http://0.0.0.0:4000/v1/audio/speech \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "azure-speech",
"input": "आप कौन हैं? चिकन डिनर क्या है?",
"voice": "en-US-AvaMultilingualNeural",
"lang": "hi-IN"
}' \
--output speech.mp3
```
### Azure-Specific Parameters Reference
| Parameter | Description | Example Values | Notes |
|-----------|-------------|----------------|-------|
| `style` | Speaking style | `cheerful`, `sad`, `angry`, `excited`, `friendly`, `hopeful`, `shouting`, `terrified`, `unfriendly`, `whispering` | Only supported by certain voices. See [Azure voice styles documentation](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-voice#use-speaking-styles-and-roles) |
| `styledegree` | Style intensity | `0.01` to `2` | Higher values = more intense. Default is `1` |
| `role` | Voice role | `Girl`, `Boy`, `YoungAdultFemale`, `YoungAdultMale`, `OlderAdultFemale`, `OlderAdultMale`, `SeniorFemale`, `SeniorMale` | Only supported by certain voices |
| `lang` | Language code | `es-ES`, `fr-FR`, `de-DE`, `hi-IN`, etc. | For multilingual voices. Overrides the default language |
## Async Support
```python showLineNumbers title="Async Usage"
import asyncio
from litellm import aspeech
from pathlib import Path
async def generate_speech():
response = await aspeech(
model="azure/speech/azure-tts",
voice="alloy",
input="Hello from async",
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
)
speech_file_path = Path(__file__).parent / "speech.mp3"
response.stream_to_file(speech_file_path)
asyncio.run(generate_speech())
```
## Regional Endpoints
Replace `{region}` with your Azure resource region:
- US East: `https://eastus.tts.speech.microsoft.com`
- US West: `https://westus.tts.speech.microsoft.com`
- Europe West: `https://westeurope.tts.speech.microsoft.com`
- Asia Southeast: `https://southeastasia.tts.speech.microsoft.com`
[Full list of regions](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/regions)
## Advanced Features
### Custom Neural Voices
You can use any Azure Neural voice by passing the full voice name:
```python showLineNumbers title="Custom Voice"
response = speech(
model="azure/speech/azure-tts",
voice="en-US-AriaNeural", # Direct Azure voice name
input="Using a specific neural voice",
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
)
```
Browse available voices in the [Azure Speech Gallery](https://speech.microsoft.com/portal/voicegallery).
## Error Handling
```python showLineNumbers title="Error Handling"
from litellm import speech
from litellm.exceptions import APIError
try:
response = speech(
model="azure/speech/azure-tts",
voice="alloy",
input="Test message",
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
)
except APIError as e:
print(f"Azure Speech error: {e}")
```
## Reference
- [Azure Speech Service Documentation](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/)
- [Text-to-Speech REST API](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/rest-text-to-speech)

245
docs/my-website/docs/providers/azure_ai_vector_stores.md Normal file
View file

@ -0,0 +1,245 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Azure AI Search - Vector Store (Unified API)
Use this to **search** Azure AI Search Vector Stores, with LiteLLM's unified `/chat/completions` API.
## Quick Start
You need three things:
1. An Azure AI Search service
2. An embedding model (to convert your queries to vectors)
3. A search index with vector fields
## Usage
<Tabs>
<TabItem value="sdk" label="SDK">
### Basic Search
```python
from litellm import vector_stores
import os
# Set your credentials
os.environ["AZURE_SEARCH_API_KEY"] = "your-search-api-key"
os.environ["AZURE_AI_SEARCH_EMBEDDING_API_BASE"] = "your-embedding-endpoint"
os.environ["AZURE_AI_SEARCH_EMBEDDING_API_KEY"] = "your-embedding-api-key"
# Search the vector store
response = vector_stores.search(
vector_store_id="my-vector-index", # Your Azure AI Search index name
query="What is the capital of France?",
custom_llm_provider="azure_ai",
azure_search_service_name="your-search-service",
litellm_embedding_model="azure/text-embedding-3-large",
litellm_embedding_config={
"api_base": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_BASE"),
"api_key": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_KEY"),
},
api_key=os.getenv("AZURE_SEARCH_API_KEY"),
)
print(response)
```
### Async Search
```python
from litellm import vector_stores
response = await vector_stores.asearch(
vector_store_id="my-vector-index",
query="What is the capital of France?",
custom_llm_provider="azure_ai",
azure_search_service_name="your-search-service",
litellm_embedding_model="azure/text-embedding-3-large",
litellm_embedding_config={
"api_base": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_BASE"),
"api_key": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_KEY"),
},
api_key=os.getenv("AZURE_SEARCH_API_KEY"),
)
print(response)
```
### Advanced Options
```python
from litellm import vector_stores
response = vector_stores.search(
vector_store_id="my-vector-index",
query="What is the capital of France?",
custom_llm_provider="azure_ai",
azure_search_service_name="your-search-service",
litellm_embedding_model="azure/text-embedding-3-large",
litellm_embedding_config={
"api_base": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_BASE"),
"api_key": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_KEY"),
},
api_key=os.getenv("AZURE_SEARCH_API_KEY"),
top_k=10, # Number of results to return
azure_search_vector_field="contentVector", # Custom vector field name
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
### Setup Config
Add this to your config.yaml:
```yaml
vector_store_registry:
- vector_store_name: "azure-ai-search-litellm-website-knowledgebase"
litellm_params:
vector_store_id: "test-litellm-app_1761094730750"
custom_llm_provider: "azure_ai"
api_key: os.environ/AZURE_SEARCH_API_KEY
litellm_embedding_model: "azure/text-embedding-3-large"
litellm_embedding_config:
api_base: https://krris-mh44uf7y-eastus2.cognitiveservices.azure.com/
api_key: os.environ/AZURE_API_KEY
api_version: "2025-09-01"
```
### Start Proxy
```bash
litellm --config /path/to/config.yaml
```
### Search via API
```bash
curl -X POST 'http://0.0.0.0:4000/v1/vector_stores/my-vector-index/search' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"query": "What is the capital of France?",
}'
```
</TabItem>
</Tabs>
## Required Parameters
| Parameter | Type | Description |
|-----------|------|-------------|
| `vector_store_id` | string | Your Azure AI Search index name |
| `custom_llm_provider` | string | Set to `"azure_ai"` |
| `azure_search_service_name` | string | Name of your Azure AI Search service |
| `litellm_embedding_model` | string | Model to generate query embeddings (e.g., `"azure/text-embedding-3-large"`) |
| `litellm_embedding_config` | dict | Config for the embedding model (api_base, api_key, api_version) |
| `api_key` | string | Your Azure AI Search API key |
## Supported Features
| Feature | Status | Notes |
|---------|--------|-------|
| Logging | ✅ Supported | Full logging support available |
| Guardrails | ❌ Not Yet Supported | Guardrails are not currently supported for vector stores |
| Cost Tracking | ✅ Supported | Cost is $0 according to Azure |
| Unified API | ✅ Supported | Call via OpenAI compatible `/v1/vector_stores/search` endpoint |
| Passthrough | ❌ Not yet supported | |
## Response Format
The response follows the standard LiteLLM vector store format:
```json
{
"object": "vector_store.search_results.page",
"search_query": "What is the capital of France?",
"data": [
{
"score": 0.95,
"content": [
{
"text": "Paris is the capital of France...",
"type": "text"
}
],
"file_id": "doc_123",
"filename": "Document doc_123",
"attributes": {
"document_id": "doc_123"
}
}
]
}
```
## How It Works
When you search:
1. LiteLLM converts your query to a vector using the embedding model you specified
2. It sends the vector to Azure AI Search
3. Azure AI Search finds the most similar documents in your index
4. Results come back with similarity scores
The embedding model can be any model supported by LiteLLM - Azure OpenAI, OpenAI, Bedrock, etc.
## Setting Up Your Azure AI Search Index
Your index needs a vector field. Here's what that looks like:
```json
{
"name": "my-vector-index",
"fields": [
{
"name": "id",
"type": "Edm.String",
"key": true
},
{
"name": "content",
"type": "Edm.String"
},
{
"name": "contentVector",
"type": "Collection(Edm.Single)",
"searchable": true,
"dimensions": 1536,
"vectorSearchProfile": "myVectorProfile"
}
]
}
```
The vector dimensions must match your embedding model. For example:
- `text-embedding-3-large`: 1536 dimensions
- `text-embedding-3-small`: 1536 dimensions
- `text-embedding-ada-002`: 1536 dimensions
## Common Issues
**"Failed to generate embedding for query"**
Your embedding model config is wrong. Check:
- `litellm_embedding_config` has the right api_base and api_key
- The embedding model name is correct
- Your credentials work
**"Index not found"**
The `vector_store_id` doesn't match any index in your search service. Check:
- The index name is correct
- You're using the right search service name
**"Field 'contentVector' not found"**
Your index uses a different vector field name. Pass it via `azure_search_vector_field`.

408
docs/my-website/docs/providers/azure_document_intelligence.md Normal file
View file

@ -0,0 +1,408 @@
# Azure Document Intelligence OCR
## Overview
| Property | Details |
|-------|-------|
| Description | Azure Document Intelligence (formerly Form Recognizer) provides advanced document analysis capabilities including text extraction, layout analysis, and structure recognition |
| Provider Route on LiteLLM | `azure_ai/doc-intelligence/` |
| Supported Operations | `/ocr` |
| Link to Provider Doc | [Azure Document Intelligence ↗](https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/)
Extract text and analyze document structure using Azure Document Intelligence's powerful prebuilt models.
## Quick Start
### **LiteLLM SDK**
```python showLineNumbers title="SDK Usage"
import litellm
import os
# Set environment variables
os.environ["AZURE_DOCUMENT_INTELLIGENCE_API_KEY"] = "your-api-key"
os.environ["AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"] = "https://your-resource.cognitiveservices.azure.com"
# OCR with PDF URL
response = litellm.ocr(
model="azure_ai/doc-intelligence/prebuilt-layout",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
)
# Access extracted text
for page in response.pages:
print(f"Page {page.index}:")
print(page.markdown)
```
### **LiteLLM PROXY**
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
- model_name: azure-doc-intel
litellm_params:
model: azure_ai/doc-intelligence/prebuilt-layout
api_key: os.environ/AZURE_DOCUMENT_INTELLIGENCE_API_KEY
api_base: os.environ/AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT
model_info:
mode: ocr
```
**Start Proxy**
```bash
litellm --config proxy_config.yaml
```
**Call OCR via Proxy**
```bash showLineNumbers title="cURL Request"
curl -X POST http://localhost:4000/ocr \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-api-key" \
-d '{
"model": "azure-doc-intel",
"document": {
"type": "document_url",
"document_url": "https://arxiv.org/pdf/2201.04234"
}
}'
```
## How It Works
Azure Document Intelligence uses an asynchronous API pattern. LiteLLM AI Gateway handles the request/response transformation and polling automatically.
### Complete Flow Diagram
```mermaid
sequenceDiagram
participant Client
box rgb(200, 220, 255) LiteLLM AI Gateway
participant LiteLLM
end
participant Azure as Azure Document Intelligence
Client->>LiteLLM: POST /ocr (Mistral format)
Note over LiteLLM: Transform to Azure format
LiteLLM->>Azure: POST :analyze
Azure-->>LiteLLM: 202 Accepted + polling URL
Note over LiteLLM: Automatic Polling
loop Every 2-10 seconds
LiteLLM->>Azure: GET polling URL
Azure-->>LiteLLM: Status: running
end
LiteLLM->>Azure: GET polling URL
Azure-->>LiteLLM: Status: succeeded + results
Note over LiteLLM: Transform to Mistral format
LiteLLM-->>Client: OCR Response (Mistral format)
```
### What LiteLLM Does For You
When you call `litellm.ocr()` via SDK or `/ocr` via Proxy:
1. **Request Transformation**: Converts Mistral OCR format → Azure Document Intelligence format
2. **Submits Document**: Sends transformed request to Azure DI API
3. **Handles 202 Response**: Captures the `Operation-Location` URL from response headers
4. **Automatic Polling**:
- Polls the operation URL at intervals specified by `retry-after` header (default: 2 seconds)
- Continues until status is `succeeded` or `failed`
- Respects Azure's rate limiting via `retry-after` headers
5. **Response Transformation**: Converts Azure DI format → Mistral OCR format
6. **Returns Result**: Sends unified Mistral format response to client
**Polling Configuration:**
- Default timeout: 120 seconds
- Configurable via `AZURE_OPERATION_POLLING_TIMEOUT` environment variable
- Uses sync (`time.sleep()`) or async (`await asyncio.sleep()`) based on call type
:::info
**Typical processing time**: 2-10 seconds depending on document size and complexity
:::
## Supported Models
Azure Document Intelligence offers several prebuilt models optimized for different use cases:
### prebuilt-layout (Recommended)
Best for general document OCR with structure preservation.
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Layout Model - SDK"
import litellm
import os
os.environ["AZURE_DOCUMENT_INTELLIGENCE_API_KEY"] = "your-api-key"
os.environ["AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"] = "https://your-resource.cognitiveservices.azure.com"
response = litellm.ocr(
model="azure_ai/doc-intelligence/prebuilt-layout",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
)
```
</TabItem>
<TabItem value="proxy" label="Proxy Config">
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
- model_name: azure-layout
litellm_params:
model: azure_ai/doc-intelligence/prebuilt-layout
api_key: os.environ/AZURE_DOCUMENT_INTELLIGENCE_API_KEY
api_base: os.environ/AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT
model_info:
mode: ocr
```
**Usage:**
```bash
curl -X POST http://localhost:4000/ocr \
-H "Authorization: Bearer your-api-key" \
-d '{"model": "azure-layout", "document": {"type": "document_url", "document_url": "https://example.com/doc.pdf"}}'
```
</TabItem>
</Tabs>
**Features:**
- Text extraction with markdown formatting
- Table detection and extraction
- Document structure analysis
- Paragraph and section recognition
**Pricing:** $10 per 1,000 pages
### prebuilt-read
Optimized for reading text from documents - fastest and most cost-effective.
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Read Model - SDK"
import litellm
import os
os.environ["AZURE_DOCUMENT_INTELLIGENCE_API_KEY"] = "your-api-key"
os.environ["AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"] = "https://your-resource.cognitiveservices.azure.com"
response = litellm.ocr(
model="azure_ai/doc-intelligence/prebuilt-read",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
)
```
</TabItem>
<TabItem value="proxy" label="Proxy Config">
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
- model_name: azure-read
litellm_params:
model: azure_ai/doc-intelligence/prebuilt-read
api_key: os.environ/AZURE_DOCUMENT_INTELLIGENCE_API_KEY
api_base: os.environ/AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT
model_info:
mode: ocr
```
**Usage:**
```bash
curl -X POST http://localhost:4000/ocr \
-H "Authorization: Bearer your-api-key" \
-d '{"model": "azure-read", "document": {"type": "document_url", "document_url": "https://example.com/doc.pdf"}}'
```
</TabItem>
</Tabs>
**Features:**
- Fast text extraction
- Optimized for reading-heavy documents
- Basic structure recognition
**Pricing:** $1.50 per 1,000 pages
### prebuilt-document
General-purpose document analysis with key-value pairs.
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Document Model - SDK"
import litellm
import os
os.environ["AZURE_DOCUMENT_INTELLIGENCE_API_KEY"] = "your-api-key"
os.environ["AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"] = "https://your-resource.cognitiveservices.azure.com"
response = litellm.ocr(
model="azure_ai/doc-intelligence/prebuilt-document",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
)
```
</TabItem>
<TabItem value="proxy" label="Proxy Config">
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
- model_name: azure-document
litellm_params:
model: azure_ai/doc-intelligence/prebuilt-document
api_key: os.environ/AZURE_DOCUMENT_INTELLIGENCE_API_KEY
api_base: os.environ/AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT
model_info:
mode: ocr
```
**Usage:**
```bash
curl -X POST http://localhost:4000/ocr \
-H "Authorization: Bearer your-api-key" \
-d '{"model": "azure-document", "document": {"type": "document_url", "document_url": "https://example.com/doc.pdf"}}'
```
</TabItem>
</Tabs>
**Pricing:** $10 per 1,000 pages
## Document Types
Azure Document Intelligence supports various document formats.
### PDF Documents
```python showLineNumbers title="PDF OCR"
response = litellm.ocr(
model="azure_ai/doc-intelligence/prebuilt-layout",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
)
```
### Image Documents
```python showLineNumbers title="Image OCR"
response = litellm.ocr(
model="azure_ai/doc-intelligence/prebuilt-layout",
document={
"type": "image_url",
"image_url": "https://example.com/image.png"
}
)
```
**Supported image formats:** JPEG, PNG, BMP, TIFF
### Base64 Encoded Documents
```python showLineNumbers title="Base64 PDF"
import base64
# Read and encode PDF
with open("document.pdf", "rb") as f:
pdf_base64 = base64.b64encode(f.read()).decode()
response = litellm.ocr(
model="azure_ai/doc-intelligence/prebuilt-layout",
document={
"type": "document_url",
"document_url": f"data:application/pdf;base64,{pdf_base64}"
}
)
```
## Response Format
```python showLineNumbers title="Response Structure"
# Response has the following structure
response.pages # List of pages with extracted text
response.model # Model used
response.object # "ocr"
response.usage_info # Token usage information
# Access page content
for page in response.pages:
print(f"Page {page.index}:")
print(page.markdown)
# Page dimensions (in pixels)
if page.dimensions:
print(f"Width: {page.dimensions.width}px")
print(f"Height: {page.dimensions.height}px")
```
## Async Support
```python showLineNumbers title="Async Usage"
import litellm
import asyncio
async def process_document():
response = await litellm.aocr(
model="azure_ai/doc-intelligence/prebuilt-layout",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
)
return response
# Run async function
response = asyncio.run(process_document())
```
## Cost Tracking
LiteLLM automatically tracks costs for Azure Document Intelligence OCR:
| Model | Cost per 1,000 Pages |
|-------|---------------------|
| prebuilt-read | $1.50 |
| prebuilt-layout | $10.00 |
| prebuilt-document | $10.00 |
```python showLineNumbers title="View Cost"
response = litellm.ocr(
model="azure_ai/doc-intelligence/prebuilt-layout",
document={"type": "document_url", "document_url": "https://..."}
)
# Access cost information
print(f"Cost: ${response._hidden_params.get('response_cost', 0)}")
```
## Additional Resources
- [Azure Document Intelligence Documentation](https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/)
- [Pricing Details](https://azure.microsoft.com/en-us/pricing/details/ai-document-intelligence/)
- [Supported File Formats](https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/concept-model-overview)
- [LiteLLM OCR Documentation](https://docs.litellm.ai/docs/ocr)

154
docs/my-website/docs/providers/azure_ocr.md Normal file
View file

@ -0,0 +1,154 @@
# Azure AI OCR (Mistral)
## Overview
| Property | Details |
|-------|-------|
| Description | Azure AI OCR provides document intelligence capabilities powered by Mistral, enabling text extraction from PDFs and images |
| Provider Route on LiteLLM | `azure_ai/` |
| Supported Operations | `/ocr` |
| Link to Provider Doc | [Azure AI ↗](https://ai.azure.com/)
Extract text from documents and images using Azure AI's OCR models, powered by Mistral.
## Quick Start
### **LiteLLM SDK**
```python showLineNumbers title="SDK Usage"
import litellm
import os
# Set environment variables
os.environ["AZURE_AI_API_KEY"] = ""
os.environ["AZURE_AI_API_BASE"] = ""
# OCR with PDF URL
response = litellm.ocr(
model="azure_ai/mistral-document-ai-2505",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
)
# Access extracted text
for page in response.pages:
print(page.text)
```
### **LiteLLM PROXY**
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
- model_name: azure-ocr
litellm_params:
model: azure_ai/mistral-document-ai-2505
api_key: "os.environ/AZURE_AI_API_KEY"
api_base: "os.environ/AZURE_AI_API_BASE"
model_info:
mode: ocr
```
## Document Types
Azure AI OCR supports both PDFs and images.
### PDF Documents
```python showLineNumbers title="PDF OCR"
response = litellm.ocr(
model="azure_ai/mistral-document-ai-2505",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
)
```
### Image Documents
```python showLineNumbers title="Image OCR"
response = litellm.ocr(
model="azure_ai/mistral-document-ai-2505",
document={
"type": "image_url",
"image_url": "https://example.com/image.png"
}
)
```
### Base64 Encoded Documents
```python showLineNumbers title="Base64 PDF"
import base64
# Read and encode PDF
with open("document.pdf", "rb") as f:
pdf_base64 = base64.b64encode(f.read()).decode()
response = litellm.ocr(
model="azure_ai/mistral-document-ai-2505",
document={
"type": "document_url",
"document_url": f"data:application/pdf;base64,{pdf_base64}"
}
)
```
## Supported Parameters
```python showLineNumbers title="All Parameters"
response = litellm.ocr(
model="azure_ai/mistral-document-ai-2505",
document={ # Required: Document to process
"type": "document_url",
"document_url": "https://..."
},
include_image_base64=True, # Optional: Include base64 images
pages=[0, 1, 2], # Optional: Specific pages to process
image_limit=10 # Optional: Limit number of images
)
```
## Response Format
```python showLineNumbers title="Response Structure"
# Response has the following structure
response.pages # List of pages with extracted text
response.model # Model used
response.object # "ocr"
response.usage_info # Token usage information
# Access page content
for page in response.pages:
print(f"Page {page.page_number}:")
print(page.text)
```
## Async Support
```python showLineNumbers title="Async Usage"
import litellm
response = await litellm.aocr(
model="azure_ai/mistral-document-ai-2505",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
)
```
## Important Notes
:::info URL Conversion
Azure AI OCR endpoints don't have internet access. LiteLLM automatically converts public URLs to base64 data URIs before sending requests to Azure AI.
:::
## Supported Models
- `mistral-document-ai-2505` - Latest Mistral OCR model on Azure AI
Use the Azure AI provider prefix: `azure_ai/<model-name>`

294
docs/my-website/docs/providers/bedrock.md
View file

@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor
| Property | Details |
|-------|-------|
| Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1) |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models) |
| Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` |
| Rerank Endpoint | `/rerank` |
@ -1734,7 +1734,69 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
</TabItem>
</Tabs>
### Qwen3 Imported Models
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/qwen3/{model_arn}` |
| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) |
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
response = completion(
model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model", # bedrock/qwen3/{your-model-arn}
messages=[{"role": "user", "content": "Tell me a joke"}],
max_tokens=100,
temperature=0.7
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: Qwen3-32B
litellm_params:
model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "Qwen3-32B", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
### OpenAI GPT OSS
@ -1937,203 +1999,13 @@ response = embedding(
### Advanced - [Pass model/provider-specific Params](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)
## Image Generation
Use this for stable diffusion, and amazon nova canvas on bedrock
See [Bedrock Image Generation](./bedrock_image_gen) for using Stable Diffusion and Amazon Nova Canvas models on Bedrock.
### Usage
## Rerank API
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import os
from litellm import image_generation
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = image_generation(
prompt="A cute baby sea otter",
model="bedrock/stability.stable-diffusion-xl-v0",
)
print(f"response: {response}")
```
**Set optional params**
```python
import os
from litellm import image_generation
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = image_generation(
prompt="A cute baby sea otter",
model="bedrock/stability.stable-diffusion-xl-v0",
### OPENAI-COMPATIBLE ###
size="128x512", # width=128, height=512
### PROVIDER-SPECIFIC ### see `AmazonStabilityConfig` in bedrock.py for all params
seed=30
)
print(f"response: {response}")
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: amazon.nova-canvas-v1:0
litellm_params:
model: bedrock/amazon.nova-canvas-v1:0
aws_region_name: "us-east-1"
aws_secret_access_key: my-key # OPTIONAL - all boto3 auth params supported
aws_secret_access_id: my-id # OPTIONAL - all boto3 auth params supported
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer $LITELLM_VIRTUAL_KEY' \
-d '{
"model": "amazon.nova-canvas-v1:0",
"prompt": "A cute baby sea otter"
}'
```
</TabItem>
</Tabs>
### Using Inference Profiles with Image Generation
For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import image_generation
response = image_generation(
model="bedrock/amazon.nova-canvas-v1:0",
model_id="arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
prompt="A cute baby sea otter"
)
print(f"response: {response}")
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: nova-canvas-inference-profile
litellm_params:
model: bedrock/amazon.nova-canvas-v1:0
model_id: arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0
aws_region_name: "eu-west-1"
```
</TabItem>
</Tabs>
## Supported AWS Bedrock Image Generation Models
| Model Name | Function Call |
|----------------------|---------------------------------------------|
| Stable Diffusion 3 - v0 | `embedding(model="bedrock/stability.stability.sd3-large-v1:0", prompt=prompt)` |
| Stable Diffusion - v0 | `embedding(model="bedrock/stability.stable-diffusion-xl-v0", prompt=prompt)` |
| Stable Diffusion - v0 | `embedding(model="bedrock/stability.stable-diffusion-xl-v1", prompt=prompt)` |
## Rerank API
Use Bedrock's Rerank API in the Cohere `/rerank` format.
Supported Cohere Rerank Params
- `model` - the foundation model ARN
- `query` - the query to rerank against
- `documents` - the list of documents to rerank
- `top_n` - the number of results to return
<Tabs>
<TabItem label="SDK" value="sdk">
```python
from litellm import rerank
import os
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = rerank(
model="bedrock/arn:aws:bedrock:us-west-2::foundation-model/amazon.rerank-v1:0", # provide the model ARN - get this here https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/bedrock/client/list_foundation_models.html
query="hello",
documents=["hello", "world"],
top_n=2,
)
print(response)
```
</TabItem>
<TabItem label="PROXY" value="proxy">
1. Setup config.yaml
```yaml
model_list:
- model_name: bedrock-rerank
litellm_params:
model: bedrock/arn:aws:bedrock:us-west-2::foundation-model/amazon.rerank-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
```
2. Start proxy server
```bash
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
```bash
curl http://0.0.0.0:4000/rerank \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "bedrock-rerank",
"query": "What is the capital of the United States?",
"documents": [
"Carson City is the capital city of the American state of Nevada.",
"The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
"Washington, D.C. is the capital of the United States.",
"Capital punishment has existed in the United States since before it was a country."
],
"top_n": 3
}'
```
</TabItem>
</Tabs>
See [Bedrock Rerank](./bedrock_rerank) for using Bedrock's Rerank API in the Cohere `/rerank` format.
## Bedrock Application Inference Profile
@ -2428,38 +2300,6 @@ model_list:
</Tabs>
Text to Image :
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer $LITELLM_VIRTUAL_KEY' \
-d '{
"model": "amazon.nova-canvas-v1:0",
"prompt": "A cute baby sea otter"
}'
```
Color Guided Generation:
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer $LITELLM_VIRTUAL_KEY' \
-d '{
"model": "amazon.nova-canvas-v1:0",
"prompt": "A cute baby sea otter",
"taskType": "COLOR_GUIDED_GENERATION",
"colorGuidedGenerationParams":{"colors":["#FFFFFF"]}
}'
```
| Model Name | Function Call |
|-------------------------|---------------------------------------------|
| Stable Diffusion 3 - v0 | `image_generation(model="bedrock/stability.stability.sd3-large-v1:0", prompt=prompt)` |
| Stable Diffusion - v0 | `image_generation(model="bedrock/stability.stable-diffusion-xl-v0", prompt=prompt)` |
| Stable Diffusion - v1 | `image_generation(model="bedrock/stability.stable-diffusion-xl-v1", prompt=prompt)` |
| Amazon Nova Canvas - v0 | `image_generation(model="bedrock/amazon.nova-canvas-v1:0", prompt=prompt)` |
### Passing an external BedrockRuntime.Client as a parameter - Completion()
This is a deprecated flow. Boto3 is not async. And boto3.client does not let us make the http call through httpx. Pass in your aws params through the method above 👆. [See Auth Code](https://github.com/BerriAI/litellm/blob/55a20c7cce99a93d36a82bf3ae90ba3baf9a7f89/litellm/llms/bedrock_httpx.py#L284) [Add new auth flow](https://github.com/BerriAI/litellm/issues)

246
docs/my-website/docs/providers/bedrock_agentcore.md Normal file
View file

@ -0,0 +1,246 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Bedrock AgentCore
Call Bedrock AgentCore in the OpenAI Request/Response format.
| Property | Details |
|----------|---------|
| Description | Amazon Bedrock AgentCore provides direct access to hosted agent runtimes for executing agentic workflows with foundation models. |
| Provider Route on LiteLLM | `bedrock/agentcore/{AGENT_RUNTIME_ARN}` |
| Provider Doc | [AWS Bedrock AgentCore ↗](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html) |
## Quick Start
### Model Format to LiteLLM
To call a bedrock agent runtime through LiteLLM, use the following model format.
Here the `model=bedrock/agentcore/` tells LiteLLM to call the bedrock `InvokeAgentRuntime` API.
```shell showLineNumbers title="Model Format to LiteLLM"
bedrock/agentcore/{AGENT_RUNTIME_ARN}
```
**Example:**
- `bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime`
You can find the Agent Runtime ARN in your AWS Bedrock console under AgentCore.
### LiteLLM Python SDK
```python showLineNumbers title="Basic AgentCore Completion"
import litellm
# Make a completion request to your AgentCore runtime
response = litellm.completion(
model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime",
messages=[
{
"role": "user",
"content": "Explain machine learning in simple terms"
}
],
)
print(response.choices[0].message.content)
print(f"Usage: {response.usage}")
```
```python showLineNumbers title="Streaming AgentCore Responses"
import litellm
# Stream responses from your AgentCore runtime
response = litellm.completion(
model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime",
messages=[
{
"role": "user",
"content": "What are the key principles of software architecture?"
}
],
stream=True,
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
```
### LiteLLM Proxy
#### 1. Configure your model in config.yaml
<Tabs>
<TabItem value="config-yaml" label="config.yaml">
```yaml showLineNumbers title="LiteLLM Proxy Configuration"
model_list:
- model_name: agentcore-runtime-1
litellm_params:
model: bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
- model_name: agentcore-runtime-2
litellm_params:
model: bedrock/agentcore/arn:aws:bedrock-agentcore:us-east-1:987654321098:runtime/production-runtime
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
```
</TabItem>
</Tabs>
#### 2. Start the LiteLLM Proxy
```bash showLineNumbers title="Start LiteLLM Proxy"
litellm --config config.yaml
```
#### 3. Make requests to your AgentCore runtimes
<Tabs>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Basic AgentCore Request"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "agentcore-runtime-1",
"messages": [
{
"role": "user",
"content": "Summarize the main benefits of cloud computing"
}
]
}'
```
```bash showLineNumbers title="Streaming AgentCore Request"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "agentcore-runtime-2",
"messages": [
{
"role": "user",
"content": "Explain the differences between SQL and NoSQL databases"
}
],
"stream": true
}'
```
</TabItem>
<TabItem value="openai-sdk" label="OpenAI Python SDK">
```python showLineNumbers title="Using OpenAI SDK with LiteLLM Proxy"
from openai import OpenAI
# Initialize client with your LiteLLM proxy URL
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-litellm-api-key"
)
# Make a completion request to your AgentCore runtime
response = client.chat.completions.create(
model="agentcore-runtime-1",
messages=[
{
"role": "user",
"content": "What are best practices for API design?"
}
]
)
print(response.choices[0].message.content)
```
```python showLineNumbers title="Streaming with OpenAI SDK"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-litellm-api-key"
)
# Stream AgentCore responses
stream = client.chat.completions.create(
model="agentcore-runtime-2",
messages=[
{
"role": "user",
"content": "Describe the microservices architecture pattern"
}
],
stream=True
)
for chunk in stream:
if chunk.choices[0].delta.content is not None:
print(chunk.choices[0].delta.content, end="")
```
</TabItem>
</Tabs>
## Provider-specific Parameters
AgentCore supports additional parameters that can be passed to customize the runtime invocation.
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Using AgentCore-specific parameters"
from litellm import completion
response = litellm.completion(
model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime",
messages=[
{
"role": "user",
"content": "Analyze this data and provide insights",
}
],
qualifier="production", # PROVIDER-SPECIFIC: Runtime qualifier/version
runtimeSessionId="session-abc-123", # PROVIDER-SPECIFIC: Custom session ID
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
```yaml showLineNumbers title="LiteLLM Proxy Configuration with Parameters"
model_list:
- model_name: agentcore-runtime-prod
litellm_params:
model: bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
qualifier: production
```
</TabItem>
</Tabs>
### Available Parameters
| Parameter | Type | Description |
|-----------|------|-------------|
| `qualifier` | string | Optional runtime qualifier/version to invoke a specific version of the agent runtime |
| `runtimeSessionId` | string | Optional custom session ID (must be 33+ characters). If not provided, LiteLLM generates one automatically |
## Further Reading
- [AWS Bedrock AgentCore Documentation](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html)
- [LiteLLM Authentication to Bedrock](https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication)

1
docs/my-website/docs/providers/bedrock_batches.md
View file

@ -9,6 +9,7 @@ Use Amazon Bedrock Batch Inference API through LiteLLM.
|----------|---------|
| Description | Amazon Bedrock Batch Inference allows you to run inference on large datasets asynchronously |
| Provider Doc | [AWS Bedrock Batch Inference ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html) |
| Cost Tracking | ✅ Supported |
## Overview

10
docs/my-website/docs/providers/bedrock_embedding.md
View file

@ -2,11 +2,11 @@
## Supported Embedding Models
| Provider | LiteLLM Route | AWS Documentation |
|----------|---------------|-------------------|
| Amazon Titan | `bedrock/amazon.*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) |
| Cohere | `bedrock/cohere.*` | [Cohere Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-embed.html) |
| TwelveLabs | `bedrock/us.twelvelabs.*` | [TwelveLabs](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-twelvelabs.html) |
| Provider | LiteLLM Route | AWS Documentation | Cost Tracking |
|----------|---------------|-------------------|---------------|
| Amazon Titan | `bedrock/amazon.*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) | ✅ |
| Cohere | `bedrock/cohere.*` | [Cohere Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-embed.html) | ✅ |
| TwelveLabs | `bedrock/us.twelvelabs.*` | [TwelveLabs](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-twelvelabs.html) | ✅ |
## Async Invoke Support

150
docs/my-website/docs/providers/bedrock_image_gen.md Normal file
View file

@ -0,0 +1,150 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# AWS Bedrock - Image Generation
Use Bedrock for image generation with Stable Diffusion, Amazon Titan Image Generator, and Amazon Nova Canvas models.
## Supported Models
| Model Name | Function Call | Cost Tracking |
|-------------------------|---------------------------------------------|---------------|
| Stable Diffusion 3 - v0 | `image_generation(model="bedrock/stability.stability.sd3-large-v1:0", prompt=prompt)` | ✅ |
| Stable Diffusion - v0 | `image_generation(model="bedrock/stability.stable-diffusion-xl-v0", prompt=prompt)` | ✅ |
| Stable Diffusion - v1 | `image_generation(model="bedrock/stability.stable-diffusion-xl-v1", prompt=prompt)` | ✅ |
| Amazon Titan Image Generator - v1 | `image_generation(model="bedrock/amazon.titan-image-generator-v1", prompt=prompt)` | ✅ |
| Amazon Titan Image Generator - v2 | `image_generation(model="bedrock/amazon.titan-image-generator-v2:0", prompt=prompt)` | ✅ |
| Amazon Nova Canvas - v1 | `image_generation(model="bedrock/amazon.nova-canvas-v1:0", prompt=prompt)` | ✅ |
## Usage
<Tabs>
<TabItem value="sdk" label="SDK">
### Basic Usage
```python
import os
from litellm import image_generation
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = image_generation(
prompt="A cute baby sea otter",
model="bedrock/stability.stable-diffusion-xl-v0",
)
print(f"response: {response}")
```
### Set Optional Parameters
```python
import os
from litellm import image_generation
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = image_generation(
prompt="A cute baby sea otter",
model="bedrock/stability.stable-diffusion-xl-v0",
### OPENAI-COMPATIBLE ###
size="128x512", # width=128, height=512
### PROVIDER-SPECIFIC ### see `AmazonStabilityConfig` in bedrock.py for all params
seed=30
)
print(f"response: {response}")
```
</TabItem>
<TabItem value="proxy" label="PROXY">
### 1. Setup config.yaml
```yaml
model_list:
- model_name: amazon.nova-canvas-v1:0
litellm_params:
model: bedrock/amazon.nova-canvas-v1:0
aws_region_name: "us-east-1"
aws_secret_access_key: my-key # OPTIONAL - all boto3 auth params supported
aws_secret_access_id: my-id # OPTIONAL - all boto3 auth params supported
```
### 2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
### 3. Test it!
**Text to Image:**
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer $LITELLM_VIRTUAL_KEY' \
-d '{
"model": "amazon.nova-canvas-v1:0",
"prompt": "A cute baby sea otter"
}'
```
**Color Guided Generation:**
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer $LITELLM_VIRTUAL_KEY' \
-d '{
"model": "amazon.nova-canvas-v1:0",
"prompt": "A cute baby sea otter",
"taskType": "COLOR_GUIDED_GENERATION",
"colorGuidedGenerationParams":{"colors":["#FFFFFF"]}
}'
```
</TabItem>
</Tabs>
## Using Inference Profiles with Image Generation
For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import image_generation
response = image_generation(
model="bedrock/amazon.nova-canvas-v1:0",
model_id="arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
prompt="A cute baby sea otter"
)
print(f"response: {response}")
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: nova-canvas-inference-profile
litellm_params:
model: bedrock/amazon.nova-canvas-v1:0
model_id: arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0
aws_region_name: "eu-west-1"
```
</TabItem>
</Tabs>
## Authentication
All standard Bedrock authentication methods are supported for image generation. See [Bedrock Authentication](./bedrock#boto3---authentication) for details.

94
docs/my-website/docs/providers/bedrock_rerank.md Normal file
View file

@ -0,0 +1,94 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# AWS Bedrock - Rerank API
Use Bedrock's Rerank API in the Cohere `/rerank` format.
:::info Cost Tracking
✅ **Cost tracking is supported** for Bedrock Rerank API calls.
:::
## Supported Parameters
- `model` - the foundation model ARN
- `query` - the query to rerank against
- `documents` - the list of documents to rerank
- `top_n` - the number of results to return
## Usage
<Tabs>
<TabItem label="SDK" value="sdk">
```python
from litellm import rerank
import os
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = rerank(
model="bedrock/arn:aws:bedrock:us-west-2::foundation-model/amazon.rerank-v1:0", # provide the model ARN - get this here https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/bedrock/client/list_foundation_models.html
query="hello",
documents=["hello", "world"],
top_n=2,
)
print(response)
```
</TabItem>
<TabItem label="PROXY" value="proxy">
### 1. Setup config.yaml
```yaml
model_list:
- model_name: bedrock-rerank
litellm_params:
model: bedrock/arn:aws:bedrock:us-west-2::foundation-model/amazon.rerank-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
```
### 2. Start proxy server
```bash
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Test it!
```bash
curl http://0.0.0.0:4000/rerank \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "bedrock-rerank",
"query": "What is the capital of the United States?",
"documents": [
"Carson City is the capital city of the American state of Nevada.",
"The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
"Washington, D.C. is the capital of the United States.",
"Capital punishment has existed in the United States since before it was a country."
],
"top_n": 3
}'
```
</TabItem>
</Tabs>
## Authentication
All standard Bedrock authentication methods are supported for rerank. See [Bedrock Authentication](./bedrock#boto3---authentication) for details.

128
docs/my-website/docs/providers/bedrock_vector_store.md
View file

@ -138,7 +138,133 @@ print(response.choices[0].message.content)
</Tabs>
Futher Reading Vector Stores:
## Filter Results
Filter by metadata attributes.
**Operators** (OpenAI-style, auto-translated):
- `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `in`, `nin`
**AWS operators** (use directly):
- `equals`, `notEquals`, `greaterThan`, `greaterThanOrEquals`, `lessThan`, `lessThanOrEquals`, `in`, `notIn`, `startsWith`, `listContains`, `stringContains`
<Tabs>
<TabItem value="single-filter" label="Single Filter">
```python
response = await litellm.acompletion(
model="anthropic/claude-3-5-sonnet",
messages=[{"role": "user", "content": "What are the latest updates?"}],
tools=[{
"type": "file_search",
"vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"],
"filters": {
"key": "category",
"value": "updates",
"operator": "eq"
}
}]
)
```
</TabItem>
<TabItem value="and-filters" label="AND">
```python
response = await litellm.acompletion(
model="anthropic/claude-3-5-sonnet",
messages=[{"role": "user", "content": "What are the policies?"}],
tools=[{
"type": "file_search",
"vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"],
"filters": {
"and": [
{"key": "category", "value": "policy", "operator": "eq"},
{"key": "year", "value": 2024, "operator": "gte"}
]
}
}]
)
```
</TabItem>
<TabItem value="or-filters" label="OR">
```python
response = await litellm.acompletion(
model="anthropic/claude-3-5-sonnet",
messages=[{"role": "user", "content": "Show me technical docs"}],
tools=[{
"type": "file_search",
"vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"],
"filters": {
"or": [
{"key": "category", "value": "api", "operator": "eq"},
{"key": "category", "value": "sdk", "operator": "eq"}
]
}
}]
)
```
</TabItem>
<TabItem value="advanced-filters" label="AWS Operators">
```python
response = await litellm.acompletion(
model="anthropic/claude-3-5-sonnet",
messages=[{"role": "user", "content": "Find docs"}],
tools=[{
"type": "file_search",
"vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"],
"filters": {
"and": [
{"key": "title", "value": "Guide", "operator": "stringContains"},
{"key": "tags", "value": "important", "operator": "listContains"}
]
}
}]
)
```
</TabItem>
<TabItem value="proxy-filters" label="Proxy">
```bash
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "claude-3-5-sonnet",
"messages": [{"role": "user", "content": "What are our policies?"}],
"tools": [{
"type": "file_search",
"vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"],
"filters": {
"and": [
{"key": "department", "value": "engineering", "operator": "eq"},
{"key": "type", "value": "policy", "operator": "eq"}
]
}
}]
}'
```
</TabItem>
</Tabs>
## Accessing Search Results
See how to access vector store search results in your response:
- [Accessing Search Results (Non-Streaming & Streaming)](../completion/knowledgebase#accessing-search-results-citations)
## Further Reading
Vector Stores:
- [Always on Vector Stores](https://docs.litellm.ai/docs/completion/knowledgebase#always-on-for-a-model)
- [Listing available vector stores on litellm proxy](https://docs.litellm.ai/docs/completion/knowledgebase#listing-available-vector-stores)
- [How LiteLLM Vector Stores Work](https://docs.litellm.ai/docs/completion/knowledgebase#how-it-works)

371
docs/my-website/docs/providers/clarifai.md
View file

@ -1,21 +1,27 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Clarifai
Anthropic, OpenAI, Mistral, Llama and Gemini LLMs are Supported on Clarifai.
Anthropic, OpenAI, Qwen, xAI, Gemini and most of Open soured LLMs are Supported on Clarifai.
:::warning
Streaming is not yet supported on using clarifai and litellm. Tracking support here: https://github.com/BerriAI/litellm/issues/4162
:::
| Property | Details |
|-------|-------|
| Description | Clarifai is a powerful AI platform that provides access to a wide range of LLMs through a unified API. LiteLLM enables seamless integration with Clarifai's models using an OpenAI-compatible interface. |
| Provider Doc | [Clarifai ↗](https://docs.clarifai.com/) |
|OpenAI compatible Endpoint for Provider | `https://api.clarifai.com/v2/ext/openai/v1` |
| Supported Endpoints | `/chat/completions` |
## Pre-Requisites
`pip install litellm`
```bash
pip install litellm
```
## Required Environment Variables
To obtain your Clarifai Personal access token follow this [link](https://docs.clarifai.com/clarifai-basics/authentication/personal-access-tokens/). Optionally the PAT can also be passed in `completion` function.
To obtain your Clarifai Personal access token follow this [link](https://docs.clarifai.com/clarifai-basics/authentication/personal-access-tokens/).
```python
os.environ["CLARIFAI_API_KEY"] = "YOUR_CLARIFAI_PAT" # CLARIFAI_PAT
os.environ["CLARIFAI_PAT"] = "CLARIFAI_API_KEY" # CLARIFAI_PAT
```
## Usage
@ -27,154 +33,231 @@ from litellm import completion
os.environ["CLARIFAI_API_KEY"] = ""
response = completion(
model="clarifai/mistralai.completion.mistral-large",
model="clarifai/openai.chat-completion.gpt-oss-20b",
messages=[{ "content": "Tell me a joke about physics?","role": "user"}]
)
```
## Streaming Support
**Output**
```json
{
"id": "chatcmpl-572701ee-9ab2-411c-ac75-46c1ba18e781",
"choices": [
{
"finish_reason": "stop",
"index": 1,
"message": {
"content": "Sure, here's a physics joke for you:\n\nWhy can't you trust an atom?\n\nBecause they make up everything!",
"role": "assistant"
}
}
LiteLLM supports streaming responses with Clarifai models:
```python
import litellm
for chunk in litellm.completion(
model="clarifai/openai.chat-completion.gpt-oss-20b",
api_key="CLARIFAI_API_KEY",
messages=[
{"role": "user", "content": "Tell me a fun fact about space."}
],
"created": 1714410197,
"model": "https://api.clarifai.com/v2/users/mistralai/apps/completion/models/mistral-large/outputs",
"object": "chat.completion",
"system_fingerprint": null,
"usage": {
"prompt_tokens": 14,
"completion_tokens": 24,
"total_tokens": 38
stream=True,
):
print(chunk.choices[0].delta)
```
## Tool Calling (Function Calling)
Clarifai models accessed via LiteLLM support function calling:
```python
import litellm
tools = [{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current temperature for a given location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "City and country e.g. Tokyo, Japan"
}
},
"required": ["location"],
"additionalProperties": False
},
}
}
}]
response = litellm.completion(
model="clarifai/openai.chat-completion.gpt-oss-20b",
api_key="CLARIFAI_API_KEY",
messages=[{"role": "user", "content": "What is the weather in Paris today?"}],
tools=tools,
)
print(response.choices[0].message.tool_calls)
```
## Clarifai models
liteLLM supports all models on [Clarifai community](https://clarifai.com/explore/models?filterData=%5B%7B%22field%22%3A%22use_cases%22%2C%22value%22%3A%5B%22llm%22%5D%7D%5D&page=1&perPage=24)
Example Usage - Note: liteLLM supports all models deployed on Clarifai
## Llama LLMs
| Model Name | Function Call |
---------------------------|---------------------------------|
| clarifai/meta.Llama-2.llama2-7b-chat | `completion('clarifai/meta.Llama-2.llama2-7b-chat', messages)`
| clarifai/meta.Llama-2.llama2-13b-chat | `completion('clarifai/meta.Llama-2.llama2-13b-chat', messages)`
| clarifai/meta.Llama-2.llama2-70b-chat | `completion('clarifai/meta.Llama-2.llama2-70b-chat', messages)` |
| clarifai/meta.Llama-2.codeLlama-70b-Python | `completion('clarifai/meta.Llama-2.codeLlama-70b-Python', messages)`|
| clarifai/meta.Llama-2.codeLlama-70b-Instruct | `completion('clarifai/meta.Llama-2.codeLlama-70b-Instruct', messages)` |
## Mistral LLMs
| Model Name | Function Call |
|---------------------------------------------|------------------------------------------------------------------------|
| clarifai/mistralai.completion.mixtral-8x22B | `completion('clarifai/mistralai.completion.mixtral-8x22B', messages)` |
| clarifai/mistralai.completion.mistral-large | `completion('clarifai/mistralai.completion.mistral-large', messages)` |
| clarifai/mistralai.completion.mistral-medium | `completion('clarifai/mistralai.completion.mistral-medium', messages)` |
| clarifai/mistralai.completion.mistral-small | `completion('clarifai/mistralai.completion.mistral-small', messages)` |
| clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1 | `completion('clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1', messages)`
| clarifai/mistralai.completion.mistral-7B-OpenOrca | `completion('clarifai/mistralai.completion.mistral-7B-OpenOrca', messages)` |
| clarifai/mistralai.completion.openHermes-2-mistral-7B | `completion('clarifai/mistralai.completion.openHermes-2-mistral-7B', messages)` |
### 🧠 OpenAI Models
- [gpt-oss-20b](https://clarifai.com/openai/chat-completion/models/gpt-oss-20b)
- [gpt-oss-120b](https://clarifai.com/openai/chat-completion/models/gpt-oss-120b)
- [gpt-5-nano](https://clarifai.com/openai/chat-completion/models/gpt-5-nano)
- [gpt-5-mini](https://clarifai.com/openai/chat-completion/models/gpt-5-mini)
- [gpt-5](https://clarifai.com/openai/chat-completion/models/gpt-5)
- [gpt-4o](https://clarifai.com/openai/chat-completion/models/gpt-4o)
- [o3](https://clarifai.com/openai/chat-completion/models/o3)
- Many more...
## Jurassic LLMs
| Model Name | Function Call |
|-----------------------------------------------|---------------------------------------------------------------------|
| clarifai/ai21.complete.Jurassic2-Grande | `completion('clarifai/ai21.complete.Jurassic2-Grande', messages)` |
| clarifai/ai21.complete.Jurassic2-Grande-Instruct | `completion('clarifai/ai21.complete.Jurassic2-Grande-Instruct', messages)` |
| clarifai/ai21.complete.Jurassic2-Jumbo-Instruct | `completion('clarifai/ai21.complete.Jurassic2-Jumbo-Instruct', messages)` |
| clarifai/ai21.complete.Jurassic2-Jumbo | `completion('clarifai/ai21.complete.Jurassic2-Jumbo', messages)` |
| clarifai/ai21.complete.Jurassic2-Large | `completion('clarifai/ai21.complete.Jurassic2-Large', messages)` |
## Wizard LLMs
| Model Name | Function Call |
|-----------------------------------------------|---------------------------------------------------------------------|
| clarifai/wizardlm.generate.wizardCoder-Python-34B | `completion('clarifai/wizardlm.generate.wizardCoder-Python-34B', messages)` |
| clarifai/wizardlm.generate.wizardLM-70B | `completion('clarifai/wizardlm.generate.wizardLM-70B', messages)` |
| clarifai/wizardlm.generate.wizardLM-13B | `completion('clarifai/wizardlm.generate.wizardLM-13B', messages)` |
| clarifai/wizardlm.generate.wizardCoder-15B | `completion('clarifai/wizardlm.generate.wizardCoder-15B', messages)` |
## Anthropic models
| Model Name | Function Call |
|-----------------------------------------------|---------------------------------------------------------------------|
| clarifai/anthropic.completion.claude-v1 | `completion('clarifai/anthropic.completion.claude-v1', messages)` |
| clarifai/anthropic.completion.claude-instant-1_2 | `completion('clarifai/anthropic.completion.claude-instant-1_2', messages)` |
| clarifai/anthropic.completion.claude-instant | `completion('clarifai/anthropic.completion.claude-instant', messages)` |
| clarifai/anthropic.completion.claude-v2 | `completion('clarifai/anthropic.completion.claude-v2', messages)` |
| clarifai/anthropic.completion.claude-2_1 | `completion('clarifai/anthropic.completion.claude-2_1', messages)` |
| clarifai/anthropic.completion.claude-3-opus | `completion('clarifai/anthropic.completion.claude-3-opus', messages)` |
| clarifai/anthropic.completion.claude-3-sonnet | `completion('clarifai/anthropic.completion.claude-3-sonnet', messages)` |
## OpenAI GPT LLMs
| Model Name | Function Call |
|-----------------------------------------------|---------------------------------------------------------------------|
| clarifai/openai.chat-completion.GPT-4 | `completion('clarifai/openai.chat-completion.GPT-4', messages)` |
| clarifai/openai.chat-completion.GPT-3_5-turbo | `completion('clarifai/openai.chat-completion.GPT-3_5-turbo', messages)` |
| clarifai/openai.chat-completion.gpt-4-turbo | `completion('clarifai/openai.chat-completion.gpt-4-turbo', messages)` |
| clarifai/openai.completion.gpt-3_5-turbo-instruct | `completion('clarifai/openai.completion.gpt-3_5-turbo-instruct', messages)` |
## GCP LLMs
| Model Name | Function Call |
|-----------------------------------------------|---------------------------------------------------------------------|
| clarifai/gcp.generate.gemini-1_5-pro | `completion('clarifai/gcp.generate.gemini-1_5-pro', messages)` |
| clarifai/gcp.generate.imagen-2 | `completion('clarifai/gcp.generate.imagen-2', messages)` |
| clarifai/gcp.generate.code-gecko | `completion('clarifai/gcp.generate.code-gecko', messages)` |
| clarifai/gcp.generate.code-bison | `completion('clarifai/gcp.generate.code-bison', messages)` |
| clarifai/gcp.generate.text-bison | `completion('clarifai/gcp.generate.text-bison', messages)` |
| clarifai/gcp.generate.gemma-2b-it | `completion('clarifai/gcp.generate.gemma-2b-it', messages)` |
| clarifai/gcp.generate.gemma-7b-it | `completion('clarifai/gcp.generate.gemma-7b-it', messages)` |
| clarifai/gcp.generate.gemini-pro | `completion('clarifai/gcp.generate.gemini-pro', messages)` |
| clarifai/gcp.generate.gemma-1_1-7b-it | `completion('clarifai/gcp.generate.gemma-1_1-7b-it', messages)` |
## Cohere LLMs
| Model Name | Function Call |
|-----------------------------------------------|---------------------------------------------------------------------|
| clarifai/cohere.generate.cohere-generate-command | `completion('clarifai/cohere.generate.cohere-generate-command', messages)` |
clarifai/cohere.generate.command-r-plus' | `completion('clarifai/clarifai/cohere.generate.command-r-plus', messages)`|
## Databricks LLMs
| Model Name | Function Call |
|---------------------------------------------------|---------------------------------------------------------------------|
| clarifai/databricks.drbx.dbrx-instruct | `completion('clarifai/databricks.drbx.dbrx-instruct', messages)` |
| clarifai/databricks.Dolly-v2.dolly-v2-12b | `completion('clarifai/databricks.Dolly-v2.dolly-v2-12b', messages)`|
## Microsoft LLMs
| Model Name | Function Call |
|---------------------------------------------------|---------------------------------------------------------------------|
| clarifai/microsoft.text-generation.phi-2 | `completion('clarifai/microsoft.text-generation.phi-2', messages)` |
| clarifai/microsoft.text-generation.phi-1_5 | `completion('clarifai/microsoft.text-generation.phi-1_5', messages)`|
## Salesforce models
| Model Name | Function Call |
|-----------------------------------------------------------|-------------------------------------------------------------------------------|
| clarifai/salesforce.blip.general-english-image-caption-blip-2 | `completion('clarifai/salesforce.blip.general-english-image-caption-blip-2', messages)` |
| clarifai/salesforce.xgen.xgen-7b-8k-instruct | `completion('clarifai/salesforce.xgen.xgen-7b-8k-instruct', messages)` |
### 🤖 Anthropic Models
- [claude-sonnet-4](https://clarifai.com/anthropic/completion/models/claude-sonnet-4)
- [claude-opus-4](https://clarifai.com/anthropic/completion/models/claude-opus-4)
- [claude-3_5-haiku](https://clarifai.com/anthropic/completion/models/claude-3_5-haiku)
- [claude-3_7-sonnet](https://clarifai.com/anthropic/completion/models/claude-3_7-sonnet)
- Many more...
## Other Top performing LLMs
### 🪄 xAI Models
- [grok-3](https://clarifai.com/xai/chat-completion/models/grok-3)
- [grok-2-vision-1212](https://clarifai.com/xai/chat-completion/models/grok-2-vision-1212)
- [grok-2-1212](https://clarifai.com/xai/chat-completion/models/grok-2-1212)
- [grok-code-fast-1](https://clarifai.com/xai/chat-completion/models/grok-code-fast-1)
- [grok-2-image-1212](https://clarifai.com/xai/image-generation/models/grok-2-image-1212)
- Many more...
| Model Name | Function Call |
|---------------------------------------------------|---------------------------------------------------------------------|
| clarifai/deci.decilm.deciLM-7B-instruct | `completion('clarifai/deci.decilm.deciLM-7B-instruct', messages)` |
| clarifai/upstage.solar.solar-10_7b-instruct | `completion('clarifai/upstage.solar.solar-10_7b-instruct', messages)` |
| clarifai/openchat.openchat.openchat-3_5-1210 | `completion('clarifai/openchat.openchat.openchat-3_5-1210', messages)` |
| clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B | `completion('clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B', messages)` |
| clarifai/fblgit.una-cybertron.una-cybertron-7b-v2 | `completion('clarifai/fblgit.una-cybertron.una-cybertron-7b-v2', messages)` |
| clarifai/tiiuae.falcon.falcon-40b-instruct | `completion('clarifai/tiiuae.falcon.falcon-40b-instruct', messages)` |
| clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat | `completion('clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat', messages)` |
| clarifai/bigcode.code.StarCoder | `completion('clarifai/bigcode.code.StarCoder', messages)` |
| clarifai/mosaicml.mpt.mpt-7b-instruct | `completion('clarifai/mosaicml.mpt.mpt-7b-instruct', messages)` |
### 🔷 Google Gemini Models
- [gemini-2_5-pro](https://clarifai.com/gcp/generate/models/gemini-2_5-pro)
- [gemini-2_5-flash-lite](https://clarifai.com/gcp/generate/models/gemini-2_5-flash-lite)
- [gemini-2_0-flash](https://clarifai.com/gcp/generate/models/gemini-2_0-flash)
- [gemini-2_0-flash-lite](https://clarifai.com/gcp/generate/models/gemini-2_0-flash-lite)
- Many more...
### 🧩 Qwen Models
- [Qwen3-30B-A3B-Instruct-2507](https://clarifai.com/qwen/qwenLM/models/Qwen3-30B-A3B-Instruct-2507)
- [Qwen3-30B-A3B-Thinking-2507](https://clarifai.com/qwen/qwenLM/models/Qwen3-30B-A3B-Thinking-2507)
- [Qwen3-14B](https://clarifai.com/qwen/qwenLM/models/Qwen3-14B)
- [QwQ-32B-AWQ](https://clarifai.com/qwen/qwenLM/models/QwQ-32B-AWQ)
- [Qwen2_5-VL-7B-Instruct](https://clarifai.com/qwen/qwen-VL/models/Qwen2_5-VL-7B-Instruct)
- [Qwen3-Coder-30B-A3B-Instruct](https://clarifai.com/qwen/qwenCoder/models/Qwen3-Coder-30B-A3B-Instruct)
- Many more...
### 💡 MiniCPM (OpenBMB) Models
- [MiniCPM-o-2_6-language](https://clarifai.com/openbmb/miniCPM/models/MiniCPM-o-2_6-language)
- [MiniCPM3-4B](https://clarifai.com/openbmb/miniCPM/models/MiniCPM3-4B)
- [MiniCPM4-8B](https://clarifai.com/openbmb/miniCPM/models/MiniCPM4-8B)
- Many more...
### 🧬 Microsoft Phi Models
- [Phi-4-reasoning-plus](https://clarifai.com/microsoft/text-generation/models/Phi-4-reasoning-plus)
- [phi-4](https://clarifai.com/microsoft/text-generation/models/phi-4)
- Many more...
### 🦙 Meta Llama Models
- [Llama-3_2-3B-Instruct](https://clarifai.com/meta/Llama-3/models/Llama-3_2-3B-Instruct)
- Many more...
### 🔍 DeepSeek Models
- [DeepSeek-R1-0528-Qwen3-8B](https://clarifai.com/deepseek-ai/deepseek-chat/models/DeepSeek-R1-0528-Qwen3-8B)
- Many more...
## Usage with LiteLLM Proxy
Here's how to call Clarifai with the LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export CLARIFAI_PAT="CLARIFAI_API_KEY"
```
### 2. Start the proxy
<Tabs>
<TabItem value="config" label="config.yaml">
```yaml
model_list:
- model_name: clarifai-model
litellm_params:
model: clarifai/openai.chat-completion.gpt-oss-20b
api_key: os.environ/CLARIFAI_PAT
```
```bash
litellm --config /path/to/config.yaml
# Server running on http://0.0.0.0:4000
```
</TabItem>
</Tabs>
### 3. Test it
<Tabs>
<TabItem value="Curl" label="Curl Request">
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data ' {
"model": "clarifai-model",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}
'
```
</TabItem>
<TabItem value="openai" label="OpenAI v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="clarifai-model",
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
]
)
print(response)
```
</TabItem>
</Tabs>
## Important Notes
- Always prefix Clarifai model IDs with `clarifai/` when specifying the model name
- Use your Clarifai Personal Access Token (PAT) as the API key
- Usage is tracked and billed through Clarifai
- API rate limits are subject to your Clarifai account settings
- Most OpenAI parameters are supported, but some advanced features may vary by model
## FAQs
| Question | Answer |
|----------|---------|
| Can I use all Clarifai models with LiteLLM? | Most chat-completion models are supported. Use the Clarifai model URL as the `model`. |
| Do I need a separate Clarifai PAT? | Yes, you must use a valid Clarifai Personal Access Token. |
| Is tool calling supported? | Yes, provided the underlying Clarifai model supports function/tool calling. |
| How is billing handled? | Clarifai usage is billed independently via Clarifai. |
## Additional Resources
- [Clarifai Documentation](https://docs.clarifai.com/)
- [LiteLLM GitHub](https://github.com/BerriAI/litellm)
- [Clarifai Runners Examples](https://github.com/Clarifai/runners-examples)

104
docs/my-website/docs/providers/cohere.md
View file

@ -15,30 +15,51 @@ os.environ["COHERE_API_KEY"] = ""
### LiteLLM Python SDK
#### Cohere v2 API (Default)
```python showLineNumbers
from litellm import completion
## set ENV variables
os.environ["COHERE_API_KEY"] = "cohere key"
# cohere call
# cohere v2 call
response = completion(
model="command-r",
model="cohere_chat/command-a-03-2025",
messages = [{ "content": "Hello, how are you?","role": "user"}]
)
```
#### Cohere v1 API
To use the Cohere v1/chat API, prefix your model name with `cohere_chat/v1/`:
```python showLineNumbers
from litellm import completion
## set ENV variables
os.environ["COHERE_API_KEY"] = "cohere key"
# cohere v1 call
response = completion(
model="cohere_chat/v1/command-a-03-2025",
messages = [{ "content": "Hello, how are you?","role": "user"}]
)
```
#### Streaming
**Cohere v2 Streaming:**
```python showLineNumbers
from litellm import completion
## set ENV variables
os.environ["COHERE_API_KEY"] = "cohere key"
# cohere call
# cohere v2 streaming
response = completion(
model="command-r",
model="cohere_chat/command-a-03-2025",
messages = [{ "content": "Hello, how are you?","role": "user"}],
stream=True
)
@ -48,6 +69,25 @@ for chunk in response:
```
**Cohere v1 Streaming:**
```python showLineNumbers
from litellm import completion
## set ENV variables
os.environ["COHERE_API_KEY"] = "cohere key"
# cohere v1 streaming
response = completion(
model="cohere_chat/v1/command-a-03-2025",
messages = [{ "content": "Hello, how are you?","role": "user"}],
stream=True
)
for chunk in response:
print(chunk)
```
## Usage with LiteLLM Proxy
@ -63,11 +103,21 @@ export COHERE_API_KEY="your-api-key"
Define the cohere models you want to use in the config.yaml
**For Cohere v1 models:**
```yaml showLineNumbers
model_list:
- model_name: command-a-03-2025
litellm_params:
model: command-a-03-2025
model: cohere_chat/v1/command-a-03-2025
api_key: "os.environ/COHERE_API_KEY"
```
**For Cohere v2 models:**
```yaml showLineNumbers
model_list:
- model_name: command-a-03-2025-v2
litellm_params:
model: cohere_chat/command-a-03-2025
api_key: "os.environ/COHERE_API_KEY"
```
@ -78,9 +128,8 @@ litellm --config /path/to/config.yaml
### 3. Test it
<Tabs>
<TabItem value="Curl" label="Curl Request">
<TabItem value="v1-curl" label="Cohere v1 - Curl Request">
```shell showLineNumbers
curl --location 'http://0.0.0.0:4000/chat/completions' \
@ -98,7 +147,25 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
'
```
</TabItem>
<TabItem value="openai" label="OpenAI v1.0.0+">
<TabItem value="v2-curl" label="Cohere v2 - Curl Request">
```shell showLineNumbers
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer <your-litellm-api-key>' \
--data ' {
"model": "command-a-03-2025-v2",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}
'
```
</TabItem>
<TabItem value="v1-openai" label="Cohere v1 - OpenAI SDK">
```python showLineNumbers
import openai
@ -107,7 +174,7 @@ client = openai.OpenAI(
base_url="http://0.0.0.0:4000"
)
# request sent to model set on litellm proxy
# request sent to cohere v1 model
response = client.chat.completions.create(model="command-a-03-2025", messages = [
{
"role": "user",
@ -116,7 +183,26 @@ response = client.chat.completions.create(model="command-a-03-2025", messages =
])
print(response)
```
</TabItem>
<TabItem value="v2-openai" label="Cohere v2 - OpenAI SDK">
```python showLineNumbers
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
# request sent to cohere v2 model
response = client.chat.completions.create(model="command-a-03-2025-v2", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
])
print(response)
```
</TabItem>
</Tabs>

4
docs/my-website/docs/providers/cometapi.md
View file

@ -1,6 +1,10 @@
# CometAPI
LiteLLM supports all AI models from [CometAPI](https://www.cometapi.com/). CometAPI provides access to 500+ AI models through a unified API interface, including cutting-edge models like GPT-5, Claude Opus 4.1, and various other state-of-the-art language models.
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_CometAPI.ipynb">
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
</a>
## Authentication
To use CometAPI models, you need to obtain an API key from [CometAPI Token Console](https://api.cometapi.com/console/token). CometAPI offers free tokens for new users - you can get your free API key instantly by registering.

69
docs/my-website/docs/providers/custom.md
View file

@ -1,69 +0,0 @@
# Custom LLM API-Endpoints
LiteLLM supports Custom deploy api endpoints
LiteLLM Expects the following input and output for custom LLM API endpoints
### Model Details
For calls to your custom API base ensure:
* Set `api_base="your-api-base"`
* Add `custom/` as a prefix to the `model` param. If your API expects `meta-llama/Llama-2-13b-hf` set `model=custom/meta-llama/Llama-2-13b-hf`
| Model Name | Function Call |
|------------------|--------------------------------------------|
| meta-llama/Llama-2-13b-hf | `response = completion(model="custom/meta-llama/Llama-2-13b-hf", messages=messages, api_base="https://your-custom-inference-endpoint")` |
| meta-llama/Llama-2-13b-hf | `response = completion(model="custom/meta-llama/Llama-2-13b-hf", messages=messages, api_base="https://api.autoai.dev/inference")` |
### Example Call to Custom LLM API using LiteLLM
```python
from litellm import completion
response = completion(
model="custom/meta-llama/Llama-2-13b-hf",
messages= [{"content": "what is custom llama?", "role": "user"}],
temperature=0.2,
max_tokens=10,
api_base="https://api.autoai.dev/inference",
request_timeout=300,
)
print("got response\n", response)
```
#### Setting your Custom API endpoint
Inputs to your custom LLM api bases should follow this format:
```python
resp = requests.post(
your-api_base,
json={
'model': 'meta-llama/Llama-2-13b-hf', # model name
'params': {
'prompt': ["The capital of France is P"],
'max_tokens': 32,
'temperature': 0.7,
'top_p': 1.0,
'top_k': 40,
}
}
)
```
Outputs from your custom LLM api bases should follow this format:
```python
{
'data': [
{
'prompt': 'The capital of France is P',
'output': [
'The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'
],
'params': {
'temperature': 0.7,
'top_k': 40,
'top_p': 1
}
}
],
'message': 'ok'
}
```

311
docs/my-website/docs/providers/fal_ai.md Normal file
View file

@ -0,0 +1,311 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Fal AI
Fal AI provides fast, scalable access to state-of-the-art image generation models including FLUX, Stable Diffusion, Imagen, and more.
## Overview
| Property | Details |
|----------|---------|
| Description | Fal AI offers optimized infrastructure for running image generation models at scale with low latency. |
| Provider Route on LiteLLM | `fal_ai/` |
| Provider Doc | [Fal AI Documentation ↗](https://fal.ai/models) |
| Supported Operations | [`/images/generations`](#image-generation) |
## Setup
### API Key
```python showLineNumbers
import os
# Set your Fal AI API key
os.environ["FAL_AI_API_KEY"] = "your-fal-api-key"
```
Get your API key from [fal.ai](https://fal.ai/).
## Supported Models
| Model Name | Description | Documentation |
|------------|-------------|---------------|
| `fal_ai/flux/schnell` | Flux Schnell - Low-latency generation with `image_size` support | [Docs ↗](https://fal.ai/models/fal-ai/flux/schnell) |
| `fal_ai/fal-ai/flux-pro/v1.1-ultra` | FLUX Pro v1.1 Ultra - High-quality image generation | [Docs ↗](https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra) |
| `fal_ai/fal-ai/imagen4/preview` | Google's Imagen 4 - Highest quality model | [Docs ↗](https://fal.ai/models/fal-ai/imagen4/preview) |
| `fal_ai/fal-ai/recraft/v3/text-to-image` | Recraft v3 - Multiple style options | [Docs ↗](https://fal.ai/models/fal-ai/recraft/v3/text-to-image) |
| `fal_ai/fal-ai/stable-diffusion-v35-medium` | Stable Diffusion v3.5 Medium | [Docs ↗](https://fal.ai/models/fal-ai/stable-diffusion-v35-medium) |
| `fal_ai/bria/text-to-image/3.2` | Bria 3.2 - Commercial-grade generation | [Docs ↗](https://fal.ai/models/bria/text-to-image/3.2) |
## Image Generation
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="basic" label="Basic Usage">
```python showLineNumbers title="Basic Image Generation"
import litellm
import os
# Set your API key
os.environ["FAL_AI_API_KEY"] = "your-fal-api-key"
# Generate an image
response = litellm.image_generation(
model="fal_ai/fal-ai/flux-pro/v1.1-ultra",
prompt="A serene mountain landscape at sunset with vibrant colors"
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="imagen4" label="Imagen 4">
```python showLineNumbers title="Google Imagen 4 Generation"
import litellm
import os
os.environ["FAL_AI_API_KEY"] = "your-fal-api-key"
# Generate with Imagen 4
response = litellm.image_generation(
model="fal_ai/fal-ai/imagen4/preview",
prompt="A vintage 1960s kitchen with flour package on countertop",
aspect_ratio="16:9",
num_images=1
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="recraft" label="Recraft v3">
```python showLineNumbers title="Recraft v3 with Style"
import litellm
import os
os.environ["FAL_AI_API_KEY"] = "your-fal-api-key"
# Generate with specific style
response = litellm.image_generation(
model="fal_ai/fal-ai/recraft/v3/text-to-image",
prompt="A red panda eating bamboo",
style="realistic_image",
image_size="landscape_4_3"
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="async" label="Async Usage">
```python showLineNumbers title="Async Image Generation"
import litellm
import asyncio
import os
async def generate_image():
os.environ["FAL_AI_API_KEY"] = "your-fal-api-key"
response = await litellm.aimage_generation(
model="fal_ai/fal-ai/stable-diffusion-v35-medium",
prompt="A cyberpunk cityscape with neon lights",
guidance_scale=7.5,
num_inference_steps=50
)
print(response.data[0].url)
return response
asyncio.run(generate_image())
```
</TabItem>
<TabItem value="advanced" label="Advanced Parameters">
```python showLineNumbers title="Advanced FLUX Pro Generation"
import litellm
import os
os.environ["FAL_AI_API_KEY"] = "your-fal-api-key"
# Generate with advanced parameters
response = litellm.image_generation(
model="fal_ai/fal-ai/flux-pro/v1.1-ultra",
prompt="A majestic dragon soaring over mountains",
n=2,
size="1792x1024", # Maps to aspect_ratio="16:9"
seed=42,
safety_tolerance="2",
enhance_prompt=True
)
for image in response.data:
print(f"Generated image: {image.url}")
```
</TabItem>
</Tabs>
### Usage - LiteLLM Proxy Server
#### 1. Configure your config.yaml
```yaml showLineNumbers title="Fal AI Image Generation Configuration"
model_list:
- model_name: flux-ultra
litellm_params:
model: fal_ai/fal-ai/flux-pro/v1.1-ultra
api_key: os.environ/FAL_AI_API_KEY
model_info:
mode: image_generation
- model_name: imagen4
litellm_params:
model: fal_ai/fal-ai/imagen4/preview
api_key: os.environ/FAL_AI_API_KEY
model_info:
mode: image_generation
- model_name: stable-diffusion
litellm_params:
model: fal_ai/fal-ai/stable-diffusion-v35-medium
api_key: os.environ/FAL_AI_API_KEY
model_info:
mode: image_generation
general_settings:
master_key: sk-1234
```
#### 2. Start LiteLLM Proxy Server
```bash showLineNumbers title="Start Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### 3. Make requests
<Tabs>
<TabItem value="openai-sdk" label="OpenAI SDK">
```python showLineNumbers title="Generate via Proxy - OpenAI SDK"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="sk-1234"
)
response = client.images.generate(
model="flux-ultra",
prompt="A beautiful sunset over the ocean",
n=1,
size="1024x1024"
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="litellm-sdk" label="LiteLLM SDK">
```python showLineNumbers title="Generate via Proxy - LiteLLM SDK"
import litellm
response = litellm.image_generation(
model="litellm_proxy/imagen4",
prompt="A cozy coffee shop interior",
api_base="http://localhost:4000",
api_key="sk-1234"
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Generate via Proxy - cURL"
curl --location 'http://localhost:4000/v1/images/generations' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "stable-diffusion",
"prompt": "A serene Japanese garden with cherry blossoms",
"n": 1,
"size": "1024x1024"
}'
```
</TabItem>
</Tabs>
## Using Model-Specific Parameters
LiteLLM forwards any additional parameters directly to the Fal AI API. You can pass model-specific parameters in your request and they will be sent to Fal AI.
```python showLineNumbers title="Pass Model-Specific Parameters"
import litellm
# Any parameters beyond the standard ones are forwarded to Fal AI
response = litellm.image_generation(
model="fal_ai/fal-ai/flux-pro/v1.1-ultra",
prompt="A beautiful sunset",
# Model-specific Fal AI parameters
aspect_ratio="16:9",
safety_tolerance="2",
enhance_prompt=True,
seed=42
)
```
For the complete list of parameters supported by each model, see:
- [FLUX Pro v1.1-ultra Parameters ↗](https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra/api)
- [Imagen 4 Parameters ↗](https://fal.ai/models/fal-ai/imagen4/preview/api)
- [Recraft v3 Parameters ↗](https://fal.ai/models/fal-ai/recraft/v3/text-to-image/api)
- [Stable Diffusion v3.5 Parameters ↗](https://fal.ai/models/fal-ai/stable-diffusion-v35-medium/api)
- [Bria 3.2 Parameters ↗](https://fal.ai/models/bria/text-to-image/3.2/api)
## Supported Parameters
Standard OpenAI-compatible parameters that work across all models:
| Parameter | Type | Description | Default |
|-----------|------|-------------|---------|
| `prompt` | string | Text description of desired image | Required |
| `model` | string | Fal AI model to use | Required |
| `n` | integer | Number of images to generate (1-4) | `1` |
| `size` | string | Image dimensions (maps to model-specific format) | Model default |
| `api_key` | string | Your Fal AI API key | Environment variable |
## Getting Started
1. Sign up at [fal.ai](https://fal.ai/)
2. Get your API key from your account settings
3. Set `FAL_AI_API_KEY` environment variable
4. Choose a model from the [Fal AI model gallery](https://fal.ai/models)
5. Start generating images with LiteLLM
## Additional Resources
- [Fal AI Documentation](https://fal.ai/docs)
- [Model Gallery](https://fal.ai/models)
- [API Reference](https://fal.ai/docs/api-reference)
- [Pricing](https://fal.ai/pricing)

6
docs/my-website/docs/providers/fireworks_ai.md
View file

@ -204,7 +204,7 @@ from litellm import completion
import os
os.environ["FIREWORKS_AI_API_KEY"] = "YOUR_API_KEY"
os.environ["FIREWORKS_AI_API_BASE"] = "https://audio-prod.us-virginia-1.direct.fireworks.ai/v1"
os.environ["FIREWORKS_AI_API_BASE"] = "https://audio-prod.api.fireworks.ai/v1"
completion = litellm.completion(
model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct",
@ -343,7 +343,7 @@ from litellm import transcription
import os
os.environ["FIREWORKS_AI_API_KEY"] = "YOUR_API_KEY"
os.environ["FIREWORKS_AI_API_BASE"] = "https://audio-prod.us-virginia-1.direct.fireworks.ai/v1"
os.environ["FIREWORKS_AI_API_BASE"] = "https://audio-prod.api.fireworks.ai/v1"
response = transcription(
model="fireworks_ai/whisper-v3",
@ -363,7 +363,7 @@ model_list:
- model_name: whisper-v3
litellm_params:
model: fireworks_ai/whisper-v3
api_base: https://audio-prod.us-virginia-1.direct.fireworks.ai/v1
api_base: https://audio-prod.api.fireworks.ai/v1
api_key: os.environ/FIREWORKS_API_KEY
model_info:
mode: audio_transcription

29
docs/my-website/docs/providers/gemini.md
View file

@ -10,7 +10,7 @@ import TabItem from '@theme/TabItem';
| Provider Route on LiteLLM | `gemini/` |
| Provider Doc | [Google AI Studio ↗](https://aistudio.google.com/) |
| API Endpoint for Provider | https://generativelanguage.googleapis.com |
| Supported OpenAI Endpoints | `/chat/completions`, [`/embeddings`](../embedding/supported_embedding#gemini-ai-embedding-models), `/completions` |
| Supported OpenAI Endpoints | `/chat/completions`, [`/embeddings`](../embedding/supported_embedding#gemini-ai-embedding-models), `/completions`, [`/videos`](./gemini/videos.md), [`/images/edits`](../image_edits.md) |
| Pass-through Endpoint | [Supported](../pass_through/google_ai_studio.md) |
<br />
@ -64,16 +64,21 @@ response = completion(
LiteLLM translates OpenAI's `reasoning_effort` to Gemini's `thinking` parameter. [Code](https://github.com/BerriAI/litellm/blob/620664921902d7a9bfb29897a7b27c1a7ef4ddfb/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py#L362)
Added an additional non-OpenAI standard "disable" value for non-reasoning Gemini requests.
**Cost Optimization:** Use `reasoning_effort="none"` (OpenAI standard) for significant cost savings - up to 96% cheaper. [Google's docs](https://ai.google.dev/gemini-api/docs/openai)
:::info
Note: Reasoning cannot be turned off on Gemini 2.5 Pro models.
:::
**Mapping**
| reasoning_effort | thinking |
| ---------------- | -------- |
| "disable" | "budget_tokens": 0 |
| "low" | "budget_tokens": 1024 |
| "medium" | "budget_tokens": 2048 |
| "high" | "budget_tokens": 4096 |
| reasoning_effort | thinking | Notes |
| ---------------- | -------- | ----- |
| "none" | "budget_tokens": 0, "includeThoughts": false | 💰 **Recommended for cost optimization** - OpenAI-compatible, always 0 |
| "disable" | "budget_tokens": DEFAULT (0), "includeThoughts": false | LiteLLM-specific, configurable via env var |
| "low" | "budget_tokens": 1024 | |
| "medium" | "budget_tokens": 2048 | |
| "high" | "budget_tokens": 4096 | |
<Tabs>
<TabItem value="sdk" label="SDK">
@ -81,6 +86,14 @@ Added an additional non-OpenAI standard "disable" value for non-reasoning Gemini
```python
from litellm import completion
# Cost-optimized: Use reasoning_effort="none" for best pricing
resp = completion(
model="gemini/gemini-2.0-flash-thinking-exp-01-21",
messages=[{"role": "user", "content": "What is the capital of France?"}],
reasoning_effort="none", # Up to 96% cheaper!
)
# Or use other levels: "low", "medium", "high"
resp = completion(
model="gemini/gemini-2.5-flash-preview-04-17",
messages=[{"role": "user", "content": "What is the capital of France?"}],

409
docs/my-website/docs/providers/gemini/videos.md Normal file
View file

@ -0,0 +1,409 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Gemini Video Generation (Veo)
LiteLLM supports Google's Veo video generation models through a unified API interface.
| Property | Details |
|-------|-------|
| Description | Google's Veo AI video generation models |
| Provider Route on LiteLLM | `gemini/` |
| Supported Models | `veo-3.0-generate-preview`, `veo-3.1-generate-preview` |
| Cost Tracking | ✅ Duration-based pricing |
| Logging Support | ✅ Full request/response logging |
| Proxy Server Support | ✅ Full proxy integration with virtual keys |
| Spend Management | ✅ Budget tracking and rate limiting |
| Link to Provider Doc | [Google Veo Documentation ↗](https://ai.google.dev/gemini-api/docs/video) |
## Quick Start
### Required API Keys
```python
import os
os.environ["GEMINI_API_KEY"] = "your-google-api-key"
# OR
os.environ["GOOGLE_API_KEY"] = "your-google-api-key"
```
### Basic Usage
```python
from litellm import video_generation, video_status, video_content
import os
import time
os.environ["GEMINI_API_KEY"] = "your-google-api-key"
# Step 1: Generate video
response = video_generation(
model="gemini/veo-3.0-generate-preview",
prompt="A cat playing with a ball of yarn in a sunny garden"
)
print(f"Video ID: {response.id}")
print(f"Initial Status: {response.status}") # "processing"
# Step 2: Poll for completion
while True:
status_response = video_status(
video_id=response.id
)
print(f"Current Status: {status_response.status}")
if status_response.status == "completed":
break
elif status_response.status == "failed":
print("Video generation failed")
break
time.sleep(10) # Wait 10 seconds before checking again
# Step 3: Download video content
video_bytes = video_content(
video_id=response.id
)
# Save to file
with open("generated_video.mp4", "wb") as f:
f.write(video_bytes)
print("Video downloaded successfully!")
```
## Supported Models
| Model Name | Description | Max Duration | Status |
|------------|-------------|--------------|--------|
| veo-3.0-generate-preview | Veo 3.0 video generation | 8 seconds | Preview |
| veo-3.1-generate-preview | Veo 3.1 video generation | 8 seconds | Preview |
## Video Generation Parameters
LiteLLM automatically maps OpenAI-style parameters to Veo's format:
| OpenAI Parameter | Veo Parameter | Description | Example |
|------------------|---------------|-------------|---------|
| `prompt` | `prompt` | Text description of the video | "A cat playing" |
| `size` | `aspectRatio` | Video dimensions → aspect ratio | "1280x720" → "16:9" |
| `seconds` | `durationSeconds` | Duration in seconds | "8" → 8 |
| `input_reference` | `image` | Reference image to animate | File object or path |
| `model` | `model` | Model to use | "gemini/veo-3.0-generate-preview" |
### Size to Aspect Ratio Mapping
LiteLLM automatically converts size dimensions to Veo's aspect ratio format:
- `"1280x720"`, `"1920x1080"` → `"16:9"` (landscape)
- `"720x1280"`, `"1080x1920"` → `"9:16"` (portrait)
### Supported Veo Parameters
Based on Veo's API:
- **prompt** (required): Text description with optional audio cues
- **aspectRatio**: `"16:9"` (default) or `"9:16"`
- **resolution**: `"720p"` (default) or `"1080p"` (Veo 3.1 only, 16:9 aspect ratio only)
- **durationSeconds**: Video length (max 8 seconds for most models)
- **image**: Reference image for animation
- **negativePrompt**: What to exclude from the video (Veo 3.1)
- **referenceImages**: Style and content references (Veo 3.1 only)
## Complete Workflow Example
```python
import litellm
import time
def generate_and_download_veo_video(
prompt: str,
output_file: str = "video.mp4",
size: str = "1280x720",
seconds: str = "8"
):
"""
Complete workflow for Veo video generation.
Args:
prompt: Text description of the video
output_file: Where to save the video
size: Video dimensions (e.g., "1280x720" for 16:9)
seconds: Duration in seconds
Returns:
bool: True if successful
"""
print(f"🎬 Generating video: {prompt}")
# Step 1: Initiate generation
response = litellm.video_generation(
model="gemini/veo-3.0-generate-preview",
prompt=prompt,
size=size, # Maps to aspectRatio
seconds=seconds # Maps to durationSeconds
)
video_id = response.id
print(f"✓ Video generation started (ID: {video_id})")
# Step 2: Wait for completion
max_wait_time = 600 # 10 minutes
start_time = time.time()
while time.time() - start_time < max_wait_time:
status_response = litellm.video_status(video_id=video_id)
if status_response.status == "completed":
print("✓ Video generation completed!")
break
elif status_response.status == "failed":
print("✗ Video generation failed")
return False
print(f"⏳ Status: {status_response.status}")
time.sleep(10)
else:
print("✗ Timeout waiting for video generation")
return False
# Step 3: Download video
print("⬇️ Downloading video...")
video_bytes = litellm.video_content(video_id=video_id)
with open(output_file, "wb") as f:
f.write(video_bytes)
print(f"✓ Video saved to {output_file}")
return True
# Use it
generate_and_download_veo_video(
prompt="A serene lake at sunset with mountains in the background",
output_file="sunset_lake.mp4"
)
```
## Async Usage
```python
from litellm import avideo_generation, avideo_status, avideo_content
import asyncio
async def async_video_workflow():
# Generate video
response = await avideo_generation(
model="gemini/veo-3.0-generate-preview",
prompt="A cat playing with a ball of yarn"
)
# Poll for completion
while True:
status = await avideo_status(video_id=response.id)
if status.status == "completed":
break
await asyncio.sleep(10)
# Download content
video_bytes = await avideo_content(video_id=response.id)
with open("video.mp4", "wb") as f:
f.write(video_bytes)
# Run it
asyncio.run(async_video_workflow())
```
## LiteLLM Proxy Usage
### Configuration
Add Veo models to your `config.yaml`:
```yaml
model_list:
- model_name: veo-3
litellm_params:
model: gemini/veo-3.0-generate-preview
api_key: os.environ/GEMINI_API_KEY
```
Start the proxy:
```bash
litellm --config config.yaml
# Server running on http://0.0.0.0:4000
```
### Making Requests
<Tabs>
<TabItem value="curl" label="Curl">
```bash
# Step 1: Generate video
curl --location 'http://0.0.0.0:4000/v1/videos' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "veo-3",
"prompt": "A cat playing with a ball of yarn in a sunny garden"
}'
# Response: {"id": "gemini::operations/generate_12345::...", "status": "processing", ...}
# Step 2: Check status
curl --location 'http://localhost:4000/v1/videos/{video_id}' \
--header 'x-litellm-api-key: sk-1234'
# Step 3: Download video (when status is "completed")
curl --location 'http://localhost:4000/v1/videos/{video_id}/content' \
--header 'x-litellm-api-key: sk-1234' \
--output video.mp4
```
</TabItem>
<TabItem value="python" label="Python SDK">
```python
import litellm
litellm.api_base = "http://0.0.0.0:4000"
litellm.api_key = "sk-1234"
# Generate video
response = litellm.video_generation(
model="veo-3",
prompt="A cat playing with a ball of yarn in a sunny garden"
)
# Check status
import time
while True:
status = litellm.video_status(video_id=response.id)
if status.status == "completed":
break
time.sleep(10)
# Download video
video_bytes = litellm.video_content(video_id=response.id)
with open("video.mp4", "wb") as f:
f.write(video_bytes)
```
</TabItem>
</Tabs>
## Cost Tracking
LiteLLM automatically tracks costs for Veo video generation:
```python
response = litellm.video_generation(
model="gemini/veo-3.0-generate-preview",
prompt="A beautiful sunset"
)
# Cost is calculated based on video duration
# Veo pricing: ~$0.10 per second (estimated)
# Default video duration: ~5 seconds
# Estimated cost: ~$0.50
```
## Differences from OpenAI Video API
| Feature | OpenAI (Sora) | Gemini (Veo) |
|---------|---------------|--------------|
| Reference Images | ✅ Supported | ❌ Not supported |
| Size Control | ✅ Supported | ❌ Not supported |
| Duration Control | ✅ Supported | ❌ Not supported |
| Video Remix/Edit | ✅ Supported | ❌ Not supported |
| Video List | ✅ Supported | ❌ Not supported |
| Prompt-based Generation | ✅ Supported | ✅ Supported |
| Async Operations | ✅ Supported | ✅ Supported |
## Error Handling
```python
from litellm import video_generation, video_status, video_content
from litellm.exceptions import APIError, Timeout
try:
response = video_generation(
model="gemini/veo-3.0-generate-preview",
prompt="A beautiful landscape"
)
# Poll with timeout
max_attempts = 60 # 10 minutes (60 * 10s)
for attempt in range(max_attempts):
status = video_status(video_id=response.id)
if status.status == "completed":
video_bytes = video_content(video_id=response.id)
with open("video.mp4", "wb") as f:
f.write(video_bytes)
break
elif status.status == "failed":
raise APIError("Video generation failed")
time.sleep(10)
else:
raise Timeout("Video generation timed out")
except APIError as e:
print(f"API Error: {e}")
except Timeout as e:
print(f"Timeout: {e}")
except Exception as e:
print(f"Unexpected error: {e}")
```
## Best Practices
1. **Always poll for completion**: Veo video generation is asynchronous and can take several minutes
2. **Set reasonable timeouts**: Allow at least 5-10 minutes for video generation
3. **Handle failures gracefully**: Check for `failed` status and implement retry logic
4. **Use descriptive prompts**: More detailed prompts generally produce better results
5. **Store video IDs**: Save the operation ID/video ID to resume polling if your application restarts
## Troubleshooting
### Video generation times out
```python
# Increase polling timeout
max_wait_time = 900 # 15 minutes instead of 10
```
### Video not found when downloading
```python
# Make sure video is completed before downloading
status = video_status(video_id=video_id)
if status.status != "completed":
print("Video not ready yet!")
```
### API key errors
```python
# Verify your API key is set
import os
print(os.environ.get("GEMINI_API_KEY"))
# Or pass it explicitly
response = video_generation(
model="gemini/veo-3.0-generate-preview",
prompt="...",
api_key="your-api-key-here"
)
```
## See Also
- [OpenAI Video Generation](../openai/videos.md)
- [Azure Video Generation](../azure/videos.md)
- [Vertex AI Video Generation](../vertex_ai/videos.md)
- [Video Generation API Reference](/docs/videos)
- [Veo Pass-through Endpoints](/docs/pass_through/google_ai_studio#example-4-video-generation-with-veo)

2
docs/my-website/docs/providers/github.md
View file

@ -1,7 +1,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# 🆕 Github
# Github
https://github.com/marketplace/models
:::tip

2
docs/my-website/docs/providers/google_ai_studio/files.md
View file

@ -39,7 +39,7 @@ encoded_string = base64.b64encode(wav_data).decode('utf-8')
file = create_file(
file=wav_data,
purpose="user_data",
extra_body={"custom_llm_provider": "gemini"},
extra_headers={"custom-llm-provider": "gemini"},
api_key=os.getenv("GEMINI_API_KEY"),
)

528
docs/my-website/docs/providers/milvus_vector_stores.md Normal file
View file

@ -0,0 +1,528 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Milvus - Vector Store
Use Milvus as a vector store for RAG.
## Quick Start
You need three things:
1. A Milvus instance (cloud or self-hosted)
2. An embedding model (to convert your queries to vectors)
3. A Milvus collection with vector fields
## Usage
<Tabs>
<TabItem value="sdk" label="SDK">
### Basic Search
```python
from litellm import vector_stores
import os
# Set your credentials
os.environ["MILVUS_API_KEY"] = "your-milvus-api-key"
os.environ["MILVUS_API_BASE"] = "https://your-milvus-instance.milvus.io"
# Search the vector store
response = vector_stores.search(
vector_store_id="my-collection-name", # Your Milvus collection name
query="What is the capital of France?",
custom_llm_provider="milvus",
litellm_embedding_model="azure/text-embedding-3-large",
litellm_embedding_config={
"api_base": "your-embedding-endpoint",
"api_key": "your-embedding-api-key",
"api_version": "2025-09-01"
},
milvus_text_field="book_intro", # Field name that contains text content
api_key=os.getenv("MILVUS_API_KEY"),
)
print(response)
```
### Async Search
```python
from litellm import vector_stores
response = await vector_stores.asearch(
vector_store_id="my-collection-name",
query="What is the capital of France?",
custom_llm_provider="milvus",
litellm_embedding_model="azure/text-embedding-3-large",
litellm_embedding_config={
"api_base": "your-embedding-endpoint",
"api_key": "your-embedding-api-key",
"api_version": "2025-09-01"
},
milvus_text_field="book_intro",
api_key=os.getenv("MILVUS_API_KEY"),
)
print(response)
```
### Advanced Options
```python
from litellm import vector_stores
response = vector_stores.search(
vector_store_id="my-collection-name",
query="What is the capital of France?",
custom_llm_provider="milvus",
litellm_embedding_model="azure/text-embedding-3-large",
litellm_embedding_config={
"api_base": "your-embedding-endpoint",
"api_key": "your-embedding-api-key",
},
milvus_text_field="book_intro",
api_key=os.getenv("MILVUS_API_KEY"),
# Milvus-specific parameters
limit=10, # Number of results to return
offset=0, # Pagination offset
dbName="default", # Database name
annsField="book_intro_vector", # Vector field name
outputFields=["id", "book_intro", "title"], # Fields to return
filter='book_id > 0', # Metadata filter expression
searchParams={"metric_type": "L2", "params": {"nprobe": 10}}, # Search parameters
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
### Setup Config
Add this to your config.yaml:
```yaml
vector_store_registry:
- vector_store_name: "milvus-knowledgebase"
litellm_params:
vector_store_id: "my-collection-name"
custom_llm_provider: "milvus"
api_key: os.environ/MILVUS_API_KEY
api_base: https://your-milvus-instance.milvus.io
litellm_embedding_model: "azure/text-embedding-3-large"
litellm_embedding_config:
api_base: https://your-endpoint.cognitiveservices.azure.com/
api_key: os.environ/AZURE_API_KEY
api_version: "2025-09-01"
milvus_text_field: "book_intro"
# Optional Milvus parameters
annsField: "book_intro_vector"
limit: 10
```
### Start Proxy
```bash
litellm --config /path/to/config.yaml
```
### Search via API
```bash
curl -X POST 'http://0.0.0.0:4000/v1/vector_stores/my-collection-name/search' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"query": "What is the capital of France?"
}'
```
</TabItem>
</Tabs>
## Required Parameters
| Parameter | Type | Description |
|-----------|------|-------------|
| `vector_store_id` | string | Your Milvus collection name |
| `custom_llm_provider` | string | Set to `"milvus"` |
| `litellm_embedding_model` | string | Model to generate query embeddings (e.g., `"azure/text-embedding-3-large"`) |
| `litellm_embedding_config` | dict | Config for the embedding model (api_base, api_key, api_version) |
| `milvus_text_field` | string | Field name in your collection that contains text content |
| `api_key` | string | Your Milvus API key (or set `MILVUS_API_KEY` env var) |
| `api_base` | string | Your Milvus API base URL (or set `MILVUS_API_BASE` env var) |
## Optional Parameters
| Parameter | Type | Description |
|-----------|------|-------------|
| `dbName` | string | Database name (default: "default") |
| `annsField` | string | Vector field name to search (default: "book_intro_vector") |
| `limit` | integer | Maximum number of results to return |
| `offset` | integer | Pagination offset |
| `filter` | string | Filter expression for metadata filtering |
| `groupingField` | string | Field to group results by |
| `outputFields` | list | List of fields to return in results |
| `searchParams` | dict | Search parameters like metric type and search parameters |
| `partitionNames` | list | List of partition names to search |
| `consistencyLevel` | string | Consistency level for the search |
## Supported Features
| Feature | Status | Notes |
|---------|--------|-------|
| Logging | ✅ Supported | Full logging support available |
| Guardrails | ❌ Not Yet Supported | Guardrails are not currently supported for vector stores |
| Cost Tracking | ✅ Supported | Cost is $0 for Milvus searches |
| Unified API | ✅ Supported | Call via OpenAI compatible `/v1/vector_stores/search` endpoint |
| Passthrough | ✅ Supported | Use native Milvus API format |
## Response Format
The response follows the standard LiteLLM vector store format:
```json
{
"object": "vector_store.search_results.page",
"search_query": "What is the capital of France?",
"data": [
{
"score": 0.95,
"content": [
{
"text": "Paris is the capital of France...",
"type": "text"
}
],
"file_id": null,
"filename": null,
"attributes": {
"id": "123",
"title": "France Geography"
}
}
]
}
```
## Passthrough API (Native Milvus Format)
Use this to allow developers to **create** and **search** vector stores using the native Milvus API format, without giving them the Milvus credentials.
This is for the proxy only.
### Admin Flow
#### 1. Add the vector store to LiteLLM
```yaml
model_list:
- model_name: embedding-model
litellm_params:
model: azure/text-embedding-3-large
api_base: https://your-endpoint.cognitiveservices.azure.com/
api_key: os.environ/AZURE_API_KEY
api_version: "2025-09-01"
vector_store_registry:
- vector_store_name: "milvus-store"
litellm_params:
vector_store_id: "can-be-anything" # vector store id can be anything for the purpose of passthrough api
custom_llm_provider: "milvus"
api_key: os.environ/MILVUS_API_KEY
api_base: https://your-milvus-instance.milvus.io
general_settings:
database_url: "postgresql://user:password@host:port/database"
master_key: "sk-1234"
```
Add your vector store credentials to LiteLLM.
#### 2. Start the proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### 3. Create a virtual index
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/indexes' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"index_name": "dall-e-6",
"litellm_params": {
"vector_store_index": "real-collection-name",
"vector_store_name": "milvus-store"
}
}'
```
This is a virtual index, which the developer can use to create and search vector stores.
#### 4. Create a key with the vector store permissions
```bash
curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"allowed_vector_store_indexes": [{"index_name": "dall-e-6", "index_permissions": ["write", "read"]}],
"models": ["embedding-model"]
}'
```
Give the key access to the virtual index and the embedding model.
**Expected response**
```json
{
"key": "sk-my-virtual-key"
}
```
### Developer Flow
#### 1. Create a collection with schema
Note: Use the `/milvus` endpoint for the passthrough api that uses the `milvus` provider in your config.
```python
from milvus_rest_client import MilvusRESTClient, DataType
import random
import time
# Configuration
uri = "http://0.0.0.0:4000/milvus" # IMPORTANT: Use the '/milvus' endpoint for passthrough
token = "sk-my-virtual-key"
collection_name = "dall-e-6" # Virtual index name
# Initialize client
milvus_client = MilvusRESTClient(uri=uri, token=token)
print(f"Connected to DB: {uri} successfully")
# Check if the collection exists and drop if it does
check_collection = milvus_client.has_collection(collection_name)
if check_collection:
milvus_client.drop_collection(collection_name)
print(f"Dropped the existing collection {collection_name} successfully")
# Define schema
dim = 64 # Vector dimension
print("Start to create the collection schema")
schema = milvus_client.create_schema()
schema.add_field(
"book_id", DataType.INT64, is_primary=True, description="customized primary id"
)
schema.add_field("word_count", DataType.INT64, description="word count")
schema.add_field(
"book_intro", DataType.FLOAT_VECTOR, dim=dim, description="book introduction"
)
# Prepare index parameters
print("Start to prepare index parameters with default AUTOINDEX")
index_params = milvus_client.prepare_index_params()
index_params.add_index("book_intro", metric_type="L2")
# Create collection
print(f"Start to create example collection: {collection_name}")
milvus_client.create_collection(
collection_name, schema=schema, index_params=index_params
)
collection_property = milvus_client.describe_collection(collection_name)
print("Collection details: %s" % collection_property)
```
#### 2. Insert data into the collection
```python
# Insert data with customized ids
nb = 1000
insert_rounds = 2
start = 0 # first primary key id
total_rt = 0 # total response time for insert
print(
f"Start to insert {nb*insert_rounds} entities into example collection: {collection_name}"
)
for i in range(insert_rounds):
vector = [random.random() for _ in range(dim)]
rows = [
{"book_id": i, "word_count": random.randint(1, 100), "book_intro": vector}
for i in range(start, start + nb)
]
t0 = time.time()
milvus_client.insert(collection_name, rows)
ins_rt = time.time() - t0
start += nb
total_rt += ins_rt
print(f"Insert completed in {round(total_rt, 4)} seconds")
# Flush the collection
print("Start to flush")
start_flush = time.time()
milvus_client.flush(collection_name)
end_flush = time.time()
print(f"Flush completed in {round(end_flush - start_flush, 4)} seconds")
```
#### 3. Search the collection
```python
# Search configuration
nq = 3 # Number of query vectors
search_params = {"metric_type": "L2", "params": {"level": 2}}
limit = 2 # Number of results to return
# Perform searches
for i in range(5):
search_vectors = [[random.random() for _ in range(dim)] for _ in range(nq)]
t0 = time.time()
results = milvus_client.search(
collection_name,
data=search_vectors,
limit=limit,
search_params=search_params,
anns_field="book_intro",
)
t1 = time.time()
print(f"Search {i} results: {results}")
print(f"Search {i} latency: {round(t1-t0, 4)} seconds")
```
#### Complete Example
Here's a full working example:
```python
from milvus_rest_client import MilvusRESTClient, DataType
import random
import time
# ----------------------------
# 🔐 CONFIGURATION
# ----------------------------
uri = "http://0.0.0.0:4000/milvus" # IMPORTANT: Use the '/milvus' endpoint
token = "sk-my-virtual-key"
collection_name = "dall-e-6" # Your virtual index name
# ----------------------------
# 📋 STEP 1 — Initialize Client
# ----------------------------
milvus_client = MilvusRESTClient(uri=uri, token=token)
print(f"✅ Connected to DB: {uri} successfully")
# ----------------------------
# 🗑️ STEP 2 — Drop Existing Collection (if needed)
# ----------------------------
check_collection = milvus_client.has_collection(collection_name)
if check_collection:
milvus_client.drop_collection(collection_name)
print(f"🗑️ Dropped the existing collection {collection_name} successfully")
# ----------------------------
# 📐 STEP 3 — Create Collection Schema
# ----------------------------
dim = 64 # Vector dimension
print("📐 Creating the collection schema")
schema = milvus_client.create_schema()
schema.add_field(
"book_id", DataType.INT64, is_primary=True, description="customized primary id"
)
schema.add_field("word_count", DataType.INT64, description="word count")
schema.add_field(
"book_intro", DataType.FLOAT_VECTOR, dim=dim, description="book introduction"
)
# ----------------------------
# 🔍 STEP 4 — Create Index
# ----------------------------
print("🔍 Preparing index parameters with default AUTOINDEX")
index_params = milvus_client.prepare_index_params()
index_params.add_index("book_intro", metric_type="L2")
# ----------------------------
# 🏗️ STEP 5 — Create Collection
# ----------------------------
print(f"🏗️ Creating collection: {collection_name}")
milvus_client.create_collection(
collection_name, schema=schema, index_params=index_params
)
collection_property = milvus_client.describe_collection(collection_name)
print(f"✅ Collection created: {collection_property}")
# ----------------------------
# 📤 STEP 6 — Insert Data
# ----------------------------
nb = 1000
insert_rounds = 2
start = 0
total_rt = 0
print(f"📤 Inserting {nb*insert_rounds} entities into collection")
for i in range(insert_rounds):
vector = [random.random() for _ in range(dim)]
rows = [
{"book_id": i, "word_count": random.randint(1, 100), "book_intro": vector}
for i in range(start, start + nb)
]
t0 = time.time()
milvus_client.insert(collection_name, rows)
ins_rt = time.time() - t0
start += nb
total_rt += ins_rt
print(f"✅ Insert completed in {round(total_rt, 4)} seconds")
# ----------------------------
# 💾 STEP 7 — Flush Collection
# ----------------------------
print("💾 Flushing collection")
start_flush = time.time()
milvus_client.flush(collection_name)
end_flush = time.time()
print(f"✅ Flush completed in {round(end_flush - start_flush, 4)} seconds")
# ----------------------------
# 🔍 STEP 8 — Search
# ----------------------------
nq = 3
search_params = {"metric_type": "L2", "params": {"level": 2}}
limit = 2
print(f"🔍 Performing {5} search operations")
for i in range(5):
search_vectors = [[random.random() for _ in range(dim)] for _ in range(nq)]
t0 = time.time()
results = milvus_client.search(
collection_name,
data=search_vectors,
limit=limit,
search_params=search_params,
anns_field="book_intro",
)
t1 = time.time()
print(f"✅ Search {i} results: {results}")
print(f" Search {i} latency: {round(t1-t0, 4)} seconds")
```
## How It Works
When you search:
1. LiteLLM converts your query to a vector using the embedding model you specified
2. It sends the vector to your Milvus instance via the `/v2/vectordb/entities/search` endpoint
3. Milvus finds the most similar documents in your collection using vector similarity search
4. Results come back with distance scores
The embedding model can be any model supported by LiteLLM - Azure OpenAI, OpenAI, Bedrock, etc.

199
docs/my-website/docs/providers/oci.md
View file

@ -29,17 +29,38 @@ Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generativ
## Authentication
LiteLLM uses OCI signing key authentication. Follow the [official Oracle tutorial](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm) to create a signing key and obtain the following parameters:
LiteLLM supports two authentication methods for OCI:
### Method 1: Manual Credentials
Provide individual OCI credentials directly to LiteLLM. Follow the [official Oracle tutorial](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm) to create a signing key and obtain the following parameters:
- `user`
- `fingerprint`
- `tenancy`
- `region`
- `key_file`
- `key_file` or `key`
- `compartment_id`
This is the default method for LiteLLM AI Gateway (LLM Proxy) access to OCI GenAI models.
### Method 2: OCI SDK Signer
Use an OCI SDK `Signer` object for authentication. This method:
- Leverages the official [OCI SDK for signing](https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html)
- Supports additional authentication methods (instance principals, workload identity, etc.)
To use this method, install the OCI SDK:
```bash
pip install oci
```
This method is an alternative when using the LiteLLM SDK on Oracle Cloud Infrastructure (instances or Oracle Kubernetes Engine).
## Usage
Input the parameters obtained from the OCI signing key creation process into the `completion` function.
<Tabs>
<TabItem value="manual" label="Manual Credentials">
Input the parameters obtained from the OCI signing key creation process into the `completion` function:
```python
import os
@ -64,10 +85,119 @@ response = completion(
print(response)
```
</TabItem>
<TabItem value="oci-sdk" label="OCI SDK Signer" default>
Use the OCI SDK `Signer` for authentication:
```python
from litellm import completion
from oci.signer import Signer
# Create an OCI Signer
signer = Signer(
tenancy="ocid1.tenancy.oc1..",
user="ocid1.user.oc1..",
fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx",
private_key_file_location="~/.oci/key.pem",
# Or use private_key_content="<your_private_key_content>"
)
messages = [{"role": "user", "content": "Hey! how's it going?"}]
response = completion(
model="oci/xai.grok-4",
messages=messages,
oci_signer=signer,
oci_region="us-chicago-1", # Optional, defaults to us-ashburn-1
oci_serving_mode="ON_DEMAND", # Optional, default is "ON_DEMAND". Other option is "DEDICATED"
oci_compartment_id="<oci_compartment_id>",
)
print(response)
```
**Alternative: Use OCI Config File**
The OCI SDK can automatically load credentials from `~/.oci/config`:
```python
from litellm import completion
from oci.config import from_file
from oci.signer import Signer
# Load config from file
config = from_file("~/.oci/config", "DEFAULT") # "DEFAULT" is the profile name
signer = Signer(
tenancy=config["tenancy"],
user=config["user"],
fingerprint=config["fingerprint"],
private_key_file_location=config["key_file"],
pass_phrase=config.get("pass_phrase") # Optional if key is encrypted
)
messages = [{"role": "user", "content": "Hey! how's it going?"}]
response = completion(
model="oci/xai.grok-4",
messages=messages,
oci_signer=signer,
oci_region=config["region"],
oci_compartment_id="<oci_compartment_id>",
)
print(response)
```
**Instance Principal Authentication**
For applications running on OCI compute instances:
```python
from litellm import completion
from oci.auth.signers import InstancePrincipalsSecurityTokenSigner
oci.auth.signers.get_oke_workload_identity_resource_principal_signer()
# Use instance principal authentication
signer = InstancePrincipalsSecurityTokenSigner()
messages = [{"role": "user", "content": "Hey! how's it going?"}]
response = completion(
model="oci/xai.grok-4",
messages=messages,
oci_signer=signer,
oci_region="us-chicago-1",
oci_compartment_id="<oci_compartment_id>",
)
print(response)
```
**Use workload identity authentication**
For applications running in Oracle Kubernetes Engine (OKE):
```python
from litellm import completion
from oci.auth.signers import get_oke_workload_identity_resource_principal_signer
# Use instance principal authentication
signer = get_oke_workload_identity_resource_principal_signer()
messages = [{"role": "user", "content": "Hey! how's it going?"}]
response = completion(
model="oci/xai.grok-4",
messages=messages,
oci_signer=signer,
oci_region="us-chicago-1",
oci_compartment_id="<oci_compartment_id>",
)
print(response)
```
</TabItem>
</Tabs>
## Usage - Streaming
Just set `stream=True` when calling completion.
<Tabs>
<TabItem value="manual-stream" label="Manual Credentials">
```python
import os
from litellm import completion
@ -93,10 +223,68 @@ for chunk in response:
print(chunk["choices"][0]["delta"]["content"]) # same as openai format
```
</TabItem>
<TabItem value="oci-sdk-stream" label="OCI SDK Signer" default>
```python
from litellm import completion
from oci.signer import Signer
signer = Signer(
tenancy="ocid1.tenancy.oc1..",
user="ocid1.user.oc1..",
fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx",
private_key_file_location="~/.oci/key.pem",
)
messages = [{"role": "user", "content": "Hey! how's it going?"}]
response = completion(
model="oci/xai.grok-4",
messages=messages,
stream=True,
oci_signer=signer,
oci_region="us-chicago-1",
oci_compartment_id="<oci_compartment_id>",
)
for chunk in response:
print(chunk["choices"][0]["delta"]["content"]) # same as openai format
```
</TabItem>
</Tabs>
## Usage Examples by Model Type
### Using Cohere Models
<Tabs>
<TabItem value="cohere-sdk" label="OCI SDK Signer" default>
```python
from litellm import completion
from oci.signer import Signer
signer = Signer(
tenancy="ocid1.tenancy.oc1..",
user="ocid1.user.oc1..",
fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx",
private_key_file_location="~/.oci/key.pem",
)
messages = [{"role": "user", "content": "Explain quantum computing"}]
response = completion(
model="oci/cohere.command-latest",
messages=messages,
oci_signer=signer,
oci_region="us-chicago-1",
oci_compartment_id="<oci_compartment_id>",
)
print(response)
```
</TabItem>
<TabItem value="cohere-manual" label="Manual Credentials">
```python
from litellm import completion
@ -112,4 +300,7 @@ response = completion(
oci_compartment_id=<oci_compartment_id>,
)
print(response)
```
```
</TabItem>
</Tabs>

83
docs/my-website/docs/providers/openai.md
View file

@ -4,6 +4,10 @@ import TabItem from '@theme/TabItem';
# OpenAI
LiteLLM supports OpenAI Chat + Embedding calls.
:::tip
**We recommend using `litellm.responses()` / Responses API** for the latest OpenAI models (GPT-5, gpt-5-codex, o3-mini, etc.)
:::
### Required API Keys
```python
@ -406,6 +410,77 @@ Expected Response:
```
### Advanced: Using `reasoning_effort` with `summary` field
By default, `reasoning_effort` accepts a string value (`"low"`, `"medium"`, `"high"`, `"minimal"`) and only sets the effort level without including a reasoning summary.
To opt-in to the `summary` feature, you can pass `reasoning_effort` as a dictionary. **Note:** The `summary` field requires your OpenAI organization to have verification status. Using `summary` without verification will result in a 400 error from OpenAI.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
# Option 1: String format (default - no summary)
response = litellm.completion(
model="openai/responses/gpt-5-mini",
messages=[{"role": "user", "content": "What is the capital of France?"}],
reasoning_effort="high" # Only sets effort level
)
# Option 2: Dict format (with optional summary - requires org verification)
response = litellm.completion(
model="openai/responses/gpt-5-mini",
messages=[{"role": "user", "content": "What is the capital of France?"}],
reasoning_effort={"effort": "high", "summary": "auto"} # "auto", "detailed", or "concise" (not all supported by all models)
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
# Option 1: String format (default - no summary)
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "openai/responses/gpt-5-mini",
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"reasoning_effort": "high"
}'
# Option 2: Dict format (with optional summary - requires org verification)
# summary options: "auto", "detailed", or "concise" (not all supported by all models)
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "openai/responses/gpt-5-mini",
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"reasoning_effort": {"effort": "high", "summary": "auto"}
}'
```
</TabItem>
</Tabs>
**Summary field options:**
- `"auto"`: System automatically determines the appropriate summary level based on the model
- `"concise"`: Provides a shorter summary (not supported by GPT-5 series models)
- `"detailed"`: Offers a comprehensive reasoning summary
**Note:** GPT-5 series models support `"auto"` and `"detailed"`, but do not support `"concise"`. O-series models (o3-pro, o4-mini, o3) support all three options. Some models like o3-mini and o1 do not support reasoning summaries at all.
**Supported `reasoning_effort` values by model:**
| Model | Default (when not set) | Supported Values |
|-------|----------------------|------------------|
| `gpt-5` | `medium` | `minimal`, `low`, `medium`, `high` |
| `gpt-5-mini` | `medium` | `minimal`, `low`, `medium`, `high` |
| `gpt-5-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |
| `gpt-5-pro` | `high` | `high` only |
**Note:** `gpt-5-pro` only accepts `reasoning_effort="high"`. Other values will return an error. When `reasoning_effort` is not set (None), OpenAI defaults to the value shown in the "Default" column.
See [OpenAI Reasoning documentation](https://platform.openai.com/docs/guides/reasoning) for more details on organization verification requirements.
## OpenAI Chat Completion to Responses API Bridge
Call any Responses API model from OpenAI's `/chat/completions` endpoint.
@ -836,4 +911,10 @@ response = completion(
model="gpt-5-pro",
messages=[{"role": "user", "content": "Solve this complex reasoning problem..."}]
)
```
```
## Video Generation
LiteLLM supports OpenAI's video generation models including Sora.
For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/video_generation.md)

12
docs/my-website/docs/providers/openai/text_to_speech.md
View file

@ -4,6 +4,18 @@ import TabItem from '@theme/TabItem';
# OpenAI - Text-to-speech
## Overview
| Feature | Supported | Notes |
|---------|-----------|-------|
| Cost Tracking | ✅ | Works with all supported models |
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input text |
| Supported Models | tts-1, tts-1-hd, gpt-4o-mini-tts | |
## **LiteLLM Python SDK Usage**
### Quick Start

247
docs/my-website/docs/providers/openai/videos.md Normal file
View file

@ -0,0 +1,247 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# OpenAI Video Generation
LiteLLM supports OpenAI's video generation models including Sora.
## Quick Start
### Required API Keys
```python
import os
os.environ["OPENAI_API_KEY"] = "your-api-key"
```
### Basic Usage
```python
from litellm import video_generation, video_content
import os
os.environ["OPENAI_API_KEY"] = "your-api-key"
# Generate a video
response = video_generation(
prompt="A cat playing with a ball of yarn in a sunny garden",
model="sora-2",
seconds="8",
size="720x1280"
)
print(f"Video ID: {response.id}")
print(f"Status: {response.status}")
# Download video content when ready
video_bytes = video_content(
video_id=response.id,
)
# Save to file
with open("generated_video.mp4", "wb") as f:
f.write(video_bytes)
```
## **LiteLLM Proxy Usage**
LiteLLM provides OpenAI API compatible video endpoints for complete video generation workflow:
- `/videos/generations` - Generate new videos
- `/videos/remix` - Edit existing videos with reference images
- `/videos/status` - Check video generation status
- `/videos/retrieval` - Download completed videos
**Setup**
Add this to your litellm proxy config.yaml
```yaml
model_list:
- model_name: sora-2
litellm_params:
model: openai/sora-2
api_key: os.environ/OPENAI_API_KEY
```
Start litellm
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
Test video generation request
```bash
curl --location 'http://localhost:4000/v1/videos' \
--header 'Content-Type: application/json' \
--header 'x-litellm-api-key: sk-1234' \
--data '{
"model": "sora-2",
"prompt": "A beautiful sunset over the ocean"
}'
```
Test video status request
```bash
# Using custom-llm-provider header
curl --location 'http://localhost:4000/v1/videos/video_id' \
--header 'Accept: application/json' \
--header 'x-litellm-api-key: sk-1234' \
--header 'custom-llm-provider: openai'
```
Test video retrieval request
```bash
# Using custom-llm-provider header
curl --location 'http://localhost:4000/v1/videos/video_id/content' \
--header 'Accept: application/json' \
--header 'x-litellm-api-key: sk-1234' \
--header 'custom-llm-provider: openai' \
--output video.mp4
# Or using query parameter
curl --location 'http://localhost:4000/v1/videos/video_id/content?custom_llm_provider=openai' \
--header 'Accept: application/json' \
--header 'x-litellm-api-key: sk-1234' \
--output video.mp4
```
Test video remix request
```bash
# Using custom_llm_provider in request body
curl --location --request POST 'http://localhost:4000/v1/videos/video_id/remix' \
--header 'Accept: application/json' \
--header 'Content-Type: application/json' \
--header 'x-litellm-api-key: sk-1234' \
--data '{
"prompt": "New remix instructions",
"custom_llm_provider": "openai"
}'
# Or using custom-llm-provider header
curl --location --request POST 'http://localhost:4000/v1/videos/video_id/remix' \
--header 'Accept: application/json' \
--header 'Content-Type: application/json' \
--header 'x-litellm-api-key: sk-1234' \
--header 'custom-llm-provider: openai' \
--data '{
"prompt": "New remix instructions"
}'
```
Test OpenAI video generation request
```bash
curl http://localhost:4000/v1/videos \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "sora-2",
"prompt": "A cat playing with a ball of yarn in a sunny garden",
"seconds": "8",
"size": "720x1280"
}'
```
## Supported Models
| Model Name | Description | Max Duration | Supported Sizes |
|------------|-------------|--------------|-----------------|
| sora-2 | OpenAI's latest video generation model | 8 seconds | 720x1280, 1280x720 |
## Video Generation Parameters
- `prompt` (required): Text description of the desired video
- `model` (optional): Model to use, defaults to "sora-2"
- `seconds` (optional): Video duration in seconds (e.g., "8", "16")
- `size` (optional): Video dimensions (e.g., "720x1280", "1280x720")
- `input_reference` (optional): Reference image for video editing
- `user` (optional): User identifier for tracking
## Video Content Retrieval
```python
# Download video content
video_bytes = video_content(
video_id="video_1234567890"
)
# Save to file
with open("video.mp4", "wb") as f:
f.write(video_bytes)
```
## Complete Workflow
```python
import litellm
import time
def generate_and_download_video(prompt):
# Step 1: Generate video
response = litellm.video_generation(
prompt=prompt,
model="sora-2",
seconds="8",
size="720x1280"
)
video_id = response.id
print(f"Video ID: {video_id}")
# Step 2: Wait for processing (in practice, poll status)
time.sleep(30)
# Step 3: Download video
video_bytes = litellm.video_content(
video_id=video_id
)
# Step 4: Save to file
with open(f"video_{video_id}.mp4", "wb") as f:
f.write(video_bytes)
return f"video_{video_id}.mp4"
# Usage
video_file = generate_and_download_video(
"A cat playing with a ball of yarn in a sunny garden"
)
```
## Video Editing with Reference Images
```python
# Video editing with reference image
response = litellm.video_generation(
prompt="Make the cat jump higher",
input_reference=open("path/to/image.jpg", "rb"), # Reference image
model="sora-2",
seconds="8"
)
print(f"Video ID: {response.id}")
```
## Error Handling
```python
from litellm.exceptions import BadRequestError, AuthenticationError
try:
response = video_generation(
prompt="A cat playing with a ball of yarn"
)
except AuthenticationError as e:
print(f"Authentication failed: {e}")
except BadRequestError as e:
print(f"Bad request: {e}")
```

33
docs/my-website/docs/providers/openrouter.md
View file

@ -9,10 +9,9 @@ LiteLLM supports all the text / chat / vision models from [OpenRouter](https://o
```python
import os
from litellm import completion
os.environ["OPENROUTER_API_KEY"] = ""
os.environ["OPENROUTER_API_BASE"] = "" # [OPTIONAL] defaults to https://openrouter.ai/api/v1
os.environ["OR_SITE_URL"] = "" # [OPTIONAL]
os.environ["OR_APP_NAME"] = "" # [OPTIONAL]
@ -22,8 +21,32 @@ response = completion(
)
```
## OpenRouter Completion Models
## Configuration with Environment Variables
For production environments, you can dynamically configure the base_url using environment variables:
```python
import os
from litellm import completion
# Configure with environment variables
OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY")
OPENROUTER_BASE_URL = os.getenv("OPENROUTER_API_BASE", "https://openrouter.ai/api/v1")
# Set environment for LiteLLM
os.environ["OPENROUTER_API_KEY"] = OPENROUTER_API_KEY
os.environ["OPENROUTER_API_BASE"] = OPENROUTER_BASE_URL
response = completion(
model="openrouter/google/palm-2-chat-bison",
messages=messages,
base_url=OPENROUTER_BASE_URL # Explicitly pass base_url for clarity
)
```
This approach provides better flexibility for managing configurations across different environments (dev, staging, production) and makes it easier to switch between self-hosted and cloud endpoints.
## OpenRouter Completion Models
🚨 LiteLLM supports ALL OpenRouter models, send `model=openrouter/<your-openrouter-model>` to send it to open router. See all openrouter models [here](https://openrouter.ai/models)
| Model Name | Function Call |
@ -40,12 +63,12 @@ response = completion(
| openrouter/meta-llama/llama-2-70b-chat | `completion('openrouter/meta-llama/llama-2-70b-chat', messages)` | `os.environ['OR_SITE_URL']`,`os.environ['OR_APP_NAME']`,`os.environ['OPENROUTER_API_KEY']` |
## Passing OpenRouter Params - transforms, models, route
Pass `transforms`, `models`, `route`as arguments to `litellm.completion()`
```python
import os
from litellm import completion
os.environ["OPENROUTER_API_KEY"] = ""
response = completion(
@ -54,4 +77,4 @@ response = completion(
transforms = [""],
route= ""
)
```
```

198
docs/my-website/docs/providers/runwayml/images.md Normal file
View file

@ -0,0 +1,198 @@
# RunwayML - Image Generation
## Overview
| Property | Details |
|-------|-------|
| Description | RunwayML provides advanced AI-powered image generation with high-quality results |
| Provider Route on LiteLLM | `runwayml/` |
| Supported Operations | [`/images/generations`](#quick-start) |
| Link to Provider Doc | [RunwayML API ↗](https://docs.dev.runwayml.com/) |
LiteLLM supports RunwayML's Gen-4 image generation API, allowing you to generate high-quality images from text prompts.
## Quick Start
```python showLineNumbers title="Basic Image Generation"
from litellm import image_generation
import os
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
response = image_generation(
model="runwayml/gen4_image",
prompt="A serene mountain landscape at sunset",
size="1920x1080"
)
print(response.data[0].url)
```
## Authentication
Set your RunwayML API key:
```python showLineNumbers title="Set API Key"
import os
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
```
## Supported Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | Model to use (e.g., `runwayml/gen4_image`) |
| `prompt` | string | Yes | Text description for the image |
| `size` | string | No | Image dimensions (default: `1920x1080`) |
### Supported Sizes
- `1024x1024`
- `1792x1024`
- `1024x1792`
- `1920x1080` (default)
- `1080x1920`
## Async Usage
```python showLineNumbers title="Async Image Generation"
from litellm import aimage_generation
import os
import asyncio
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
async def generate_image():
response = await aimage_generation(
model="runwayml/gen4_image",
prompt="A futuristic city skyline at night",
size="1920x1080"
)
print(response.data[0].url)
asyncio.run(generate_image())
```
## LiteLLM Proxy Usage
Add RunwayML to your proxy configuration:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gen4-image
litellm_params:
model: runwayml/gen4_image
api_key: os.environ/RUNWAYML_API_KEY
```
Start the proxy:
```bash
litellm --config /path/to/config.yaml
```
Generate images through the proxy:
```bash showLineNumbers title="Proxy Request"
curl --location 'http://localhost:4000/v1/images/generations' \
--header 'Content-Type: application/json' \
--header 'x-litellm-api-key: sk-1234' \
--data '{
"model": "runwayml/gen4_image",
"prompt": "A serene mountain landscape at sunset",
"size": "1920x1080"
}'
```
## Supported Models
| Model | Description | Default Size |
|-------|-------------|--------------|
| `runwayml/gen4_image` | High-quality image generation | 1920x1080 |
## Cost Tracking
LiteLLM automatically tracks RunwayML image generation costs:
```python showLineNumbers title="Cost Tracking"
from litellm import image_generation, completion_cost
response = image_generation(
model="runwayml/gen4_image",
prompt="A serene mountain landscape at sunset",
size="1920x1080"
)
cost = completion_cost(completion_response=response)
print(f"Image generation cost: ${cost}")
```
## Supported Features
| Feature | Supported |
|---------|-----------|
| Image Generation | ✅ |
| Cost Tracking | ✅ |
| Logging | ✅ |
| Fallbacks | ✅ |
| Load Balancing | ✅ |
## How It Works
RunwayML uses an asynchronous task-based API pattern. LiteLLM handles the polling and response transformation automatically.
### Complete Flow Diagram
```mermaid
sequenceDiagram
participant Client
box rgb(200, 220, 255) LiteLLM AI Gateway
participant LiteLLM
end
participant RunwayML as RunwayML API
Client->>LiteLLM: POST /images/generations (OpenAI format)
Note over LiteLLM: Transform to RunwayML format
LiteLLM->>RunwayML: POST v1/text_to_image
RunwayML-->>LiteLLM: 200 OK + task ID
Note over LiteLLM: Automatic Polling
loop Every 2 seconds
LiteLLM->>RunwayML: GET v1/tasks/{task_id}
RunwayML-->>LiteLLM: Status: RUNNING
end
LiteLLM->>RunwayML: GET v1/tasks/{task_id}
RunwayML-->>LiteLLM: Status: SUCCEEDED + image URL
Note over LiteLLM: Transform to OpenAI format
LiteLLM-->>Client: Image Response (OpenAI format)
```
### What LiteLLM Does For You
When you call `litellm.image_generation()` or `/v1/images/generations`:
1. **Request Transformation**: Converts OpenAI image generation format → RunwayML format
2. **Submits Task**: Sends transformed request to RunwayML API
3. **Receives Task ID**: Captures the task ID from the initial response
4. **Automatic Polling**:
- Polls the task status endpoint every 2 seconds
- Continues until status is `SUCCEEDED` or `FAILED`
- Default timeout: 10 minutes (configurable via `RUNWAYML_POLLING_TIMEOUT`)
5. **Response Transformation**: Converts RunwayML format → OpenAI format
6. **Returns Result**: Sends unified OpenAI format response to client
**Polling Configuration:**
- Default timeout: 600 seconds (10 minutes)
- Configurable via `RUNWAYML_POLLING_TIMEOUT` environment variable
- Uses sync (`time.sleep()`) or async (`await asyncio.sleep()`) based on call type
:::info
**Typical processing time**: 10-30 seconds depending on image size and complexity
:::

244
docs/my-website/docs/providers/runwayml/text-to-speech.md Normal file
View file

@ -0,0 +1,244 @@
# RunwayML - Text-to-Speech
## Overview
| Property | Details |
|-------|-------|
| Description | RunwayML provides high-quality AI-powered text-to-speech with natural-sounding voices |
| Provider Route on LiteLLM | `runwayml/` |
| Supported Operations | [`/audio/speech`](#quick-start) |
| Link to Provider Doc | [RunwayML API ↗](https://docs.dev.runwayml.com/) |
LiteLLM supports RunwayML's text-to-speech API with automatic task polling, allowing you to generate natural-sounding audio from text.
## Quick Start
```python showLineNumbers title="Basic Text-to-Speech"
from litellm import speech
import os
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
response = speech(
model="runwayml/eleven_multilingual_v2",
input="Step right up, ladies and gentlemen! Have you ever wished for a toaster that's not just a toaster but a marvel of modern ingenuity?",
voice="alloy"
)
# Save the audio
with open("output.mp3", "wb") as f:
f.write(response.content)
```
## Authentication
Set your RunwayML API key:
```python showLineNumbers title="Set API Key"
import os
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
```
## Supported Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | Model to use (e.g., `runwayml/eleven_multilingual_v2`) |
| `input` | string | Yes | Text to convert to speech |
| `voice` | string or dict | Yes | Voice to use (OpenAI name, RunwayML preset, or voice config) |
## Voice Options
### Using OpenAI Voice Names
OpenAI voice names are automatically mapped to appropriate RunwayML voices:
```python showLineNumbers title="OpenAI Voice Names"
from litellm import speech
# These OpenAI voice names work automatically
response = speech(
model="runwayml/eleven_multilingual_v2",
input="Hello, world!",
voice="alloy" # Maya - neutral, balanced female voice
)
```
**Voice Mappings:**
- `alloy` → Maya (neutral, balanced female voice)
- `echo` → James (male voice)
- `fable` → Bernard (warm, storytelling voice)
- `onyx` → Vincent (deep male voice)
- `nova` → Serene (warm, expressive female voice)
- `shimmer` → Ella (clear, friendly female voice)
### Using RunwayML Preset Voices
You can directly specify any RunwayML preset voice by passing the preset name as a string:
```python showLineNumbers title="RunwayML Preset Names"
from litellm import speech
# Pass the RunwayML voice name as a string
response = speech(
model="runwayml/eleven_multilingual_v2",
input="Hello, world!",
voice="Maya" # LiteLLM automatically formats this for RunwayML
)
# Try different RunwayML voices
response = speech(
model="runwayml/eleven_multilingual_v2",
input="Step right up, ladies and gentlemen!",
voice="Bernard" # Great for storytelling
)
```
**Available RunwayML Voices:**
Maya, Arjun, Serene, Bernard, Billy, Mark, Clint, Mabel, Chad, Leslie, Eleanor, Elias, Elliot, Grungle, Brodie, Sandra, Kirk, Kylie, Lara, Lisa, Malachi, Marlene, Martin, Miriam, Monster, Paula, Pip, Rusty, Ragnar, Xylar, Maggie, Jack, Katie, Noah, James, Rina, Ella, Mariah, Frank, Claudia, Niki, Vincent, Kendrick, Myrna, Tom, Wanda, Benjamin, Kiana, Rachel
:::tip
Simply pass the voice name as a string - LiteLLM automatically handles the internal RunwayML API format conversion.
:::
## Async Usage
```python showLineNumbers title="Async Text-to-Speech"
from litellm import aspeech
import os
import asyncio
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
async def generate_speech():
response = await aspeech(
model="runwayml/eleven_multilingual_v2",
input="This is an asynchronous text-to-speech request.",
voice="nova"
)
with open("output.mp3", "wb") as f:
f.write(response.content)
print("Audio generated successfully!")
asyncio.run(generate_speech())
```
## LiteLLM Proxy Usage
Add RunwayML to your proxy configuration:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: runway-tts
litellm_params:
model: runwayml/eleven_multilingual_v2
api_key: os.environ/RUNWAYML_API_KEY
```
Start the proxy:
```bash
litellm --config /path/to/config.yaml
```
Generate speech through the proxy:
```bash showLineNumbers title="Proxy Request"
curl --location 'http://localhost:4000/v1/audio/speech' \
--header 'Content-Type: application/json' \
--header 'x-litellm-api-key: sk-1234' \
--data '{
"model": "runwayml/eleven_multilingual_v2",
"input": "Hello from the LiteLLM proxy!",
"voice": "alloy"
}'
```
With RunwayML-specific voice:
```bash showLineNumbers title="Proxy Request with RunwayML Voice"
curl --location 'http://localhost:4000/v1/audio/speech' \
--header 'Content-Type: application/json' \
--header 'x-litellm-api-key: sk-1234' \
--data '{
"model": "runwayml/eleven_multilingual_v2",
"input": "Hello with a custom RunwayML voice!",
"voice": "Bernard"
}'
```
## Supported Models
| Model | Description |
|-------|-------------|
| `runwayml/eleven_multilingual_v2` | High-quality multilingual text-to-speech |
## Cost Tracking
LiteLLM automatically tracks RunwayML text-to-speech costs:
```python showLineNumbers title="Cost Tracking"
from litellm import speech, completion_cost
response = speech(
model="runwayml/eleven_multilingual_v2",
input="Hello, world!",
voice="alloy"
)
cost = completion_cost(completion_response=response)
print(f"Text-to-speech cost: ${cost}")
```
## Supported Features
| Feature | Supported |
|---------|-----------|
| Text-to-Speech | ✅ |
| Cost Tracking | ✅ |
| Logging | ✅ |
| Fallbacks | ✅ |
| Load Balancing | ✅ |
| 50+ Voice Presets | ✅ |
## How It Works
RunwayML uses an asynchronous task-based API pattern. LiteLLM handles the polling and response transformation automatically.
### Complete Flow Diagram
```mermaid
sequenceDiagram
participant Client
box rgb(200, 220, 255) LiteLLM AI Gateway
participant LiteLLM
end
participant RunwayML as RunwayML API
participant Storage as Audio Storage
Client->>LiteLLM: POST /audio/speech (OpenAI format)
Note over LiteLLM: Transform to RunwayML format<br/>Map voice to preset ID
LiteLLM->>RunwayML: POST v1/text_to_speech
RunwayML-->>LiteLLM: 200 OK + task ID
Note over LiteLLM: Automatic Polling
loop Every 2 seconds
LiteLLM->>RunwayML: GET v1/tasks/{task_id}
RunwayML-->>LiteLLM: Status: RUNNING
end
LiteLLM->>RunwayML: GET v1/tasks/{task_id}
RunwayML-->>LiteLLM: Status: SUCCEEDED + audio URL
LiteLLM->>Storage: GET audio URL
Storage-->>LiteLLM: Audio data (MP3)
Note over LiteLLM: Return audio content
LiteLLM-->>Client: Audio Response (binary)
```

266
docs/my-website/docs/providers/runwayml/videos.md Normal file
View file

@ -0,0 +1,266 @@
# RunwayML - Video Generation
LiteLLM supports RunwayML's Gen-4 video generation API, allowing you to generate videos from text prompts and images.
## Quick Start
```python showLineNumbers title="Basic Video Generation"
from litellm import video_generation
import os
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
# Generate video from text and image
response = video_generation(
model="runwayml/gen4_turbo",
prompt="A high quality demo video of litellm ai gateway",
input_reference="https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY",
seconds=5,
size="1280x720"
)
print(f"Video ID: {response.id}")
print(f"Status: {response.status}")
```
## Authentication
Set your RunwayML API key:
```python showLineNumbers title="Set API Key"
import os
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
```
## Supported Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | Model to use (e.g., `runwayml/gen4_turbo`) |
| `prompt` | string | Yes | Text description for the video |
| `input_reference` | string/file | Yes | URL or file path to reference image |
| `seconds` | int | No | Video duration (5 or 10 seconds) |
| `size` | string | No | Video dimensions (`1280x720` or `720x1280`). Can also use `ratio` format (`1280:720`) |
## Complete Workflow
```python showLineNumbers title="Complete Video Generation Workflow"
from litellm import video_generation, video_status, video_content
import os
import time
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
# 1. Generate video
response = video_generation(
model="runwayml/gen4_turbo",
prompt="A high quality demo video of litellm ai gateway",
input_reference="https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY",
seconds=5,
size="1280x720"
)
video_id = response.id
print(f"Video generation started: {video_id}")
# 2. Check status until completed
while True:
status_response = video_status(video_id=video_id)
print(f"Status: {status_response.status}")
if status_response.status == "completed":
print("Video generation completed!")
break
elif status_response.status == "failed":
print("Video generation failed")
break
time.sleep(10) # Wait 10 seconds before checking again
# 3. Download video content
video_bytes = video_content(video_id=video_id)
# 4. Save to file
with open("generated_video.mp4", "wb") as f:
f.write(video_bytes)
print("Video saved successfully!")
```
## Async Usage
```python showLineNumbers title="Async Video Generation"
from litellm import avideo_generation, avideo_status, avideo_content
import os
import asyncio
os.environ["RUNWAYML_API_KEY"] = "your-api-key"
async def generate_video():
# Generate video
response = await avideo_generation(
model="runwayml/gen4_turbo",
prompt="A serene lake with mountains in the background",
input_reference="https://example.com/lake.jpg",
seconds=5,
size="1280x720"
)
video_id = response.id
print(f"Video generation started: {video_id}")
# Poll for completion
while True:
status_response = await avideo_status(video_id=video_id)
print(f"Status: {status_response.status}")
if status_response.status == "completed":
break
elif status_response.status == "failed":
print("Video generation failed")
return
await asyncio.sleep(10)
# Download video
video_bytes = await avideo_content(video_id=video_id)
# Save to file
with open("generated_video.mp4", "wb") as f:
f.write(video_bytes)
print("Video saved successfully!")
asyncio.run(generate_video())
```
## LiteLLM Proxy Usage
Add RunwayML to your proxy configuration:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gen4-turbo
litellm_params:
model: runwayml/gen4_turbo
api_key: os.environ/RUNWAYML_API_KEY
```
Start the proxy:
```bash
litellm --config /path/to/config.yaml
```
Generate videos through the proxy:
```bash showLineNumbers title="Proxy Request"
curl --location 'http://localhost:4000/v1/videos' \
--header 'Content-Type: application/json' \
--header 'x-litellm-api-key: sk-1234' \
--data '{
"model": "runwayml/gen4_turbo",
"prompt": "A high quality demo video of litellm ai gateway",
"input_reference": "https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY",
"ratio": "1280:720"
}'
```
Check video status:
```bash showLineNumbers title="Check Status"
curl --location 'http://localhost:4000/v1/videos/{video_id}' \
--header 'x-litellm-api-key: sk-1234'
```
Download video content:
```bash showLineNumbers title="Download Video"
curl --location 'http://localhost:4000/v1/videos/{video_id}/content' \
--header 'x-litellm-api-key: sk-1234' \
--output video.mp4
```
## Supported Models
| Model | Description | Duration | Aspect Ratios |
|-------|-------------|----------|---------------|
| `runwayml/gen4_turbo` | Fast video generation | 5-10s | 1280x720, 720x1280 |
## Error Handling
```python showLineNumbers title="Error Handling"
from litellm import video_generation, video_status
import time
try:
response = video_generation(
model="runwayml/gen4_turbo",
prompt="A scenic mountain view",
input_reference="https://example.com/mountain.jpg",
seconds=5
)
# Poll for completion
max_attempts = 60 # 10 minutes max
attempts = 0
while attempts < max_attempts:
status_response = video_status(video_id=response.id)
if status_response.status == "completed":
print("Video generation completed!")
break
elif status_response.status == "failed":
error = status_response.error or {}
print(f"Video generation failed: {error.get('message', 'Unknown error')}")
break
time.sleep(10)
attempts += 1
if attempts >= max_attempts:
print("Video generation timed out")
except Exception as e:
print(f"Error: {str(e)}")
```
## Cost Tracking
LiteLLM automatically tracks RunwayML video generation costs:
```python showLineNumbers title="Cost Tracking"
from litellm import video_generation, completion_cost
response = video_generation(
model="runwayml/gen4_turbo",
prompt="A high quality demo video of litellm ai gateway",
input_reference="https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY",
seconds=5,
size="1280x720"
)
# Calculate cost
cost = completion_cost(completion_response=response)
print(f"Video generation cost: ${cost}")
```
## API Reference
For complete API details, see the [OpenAI Video Generation API specification](https://platform.openai.com/docs/guides/video-generation) which LiteLLM follows.
## Supported Features
| Feature | Supported |
|---------|-----------|
| Video Generation | ✅ |
| Image-to-Video | ✅ |
| Status Checking | ✅ |
| Content Download | ✅ |
| Cost Tracking | ✅ |
| Logging | ✅ |
| Fallbacks | ✅ |
| Load Balancing | ✅ |

666
docs/my-website/docs/providers/vertex.md
View file

@ -12,7 +12,7 @@ import TabItem from '@theme/TabItem';
| Provider Route on LiteLLM | `vertex_ai/` |
| Link to Provider Doc | [Vertex AI ↗](https://cloud.google.com/vertex-ai) |
| Base URL | 1. Regional endpoints<br/>`https://{vertex_location}-aiplatform.googleapis.com/`<br/>2. Global endpoints (limited availability)<br/>`https://aiplatform.googleapis.com/`|
| Supported Operations | [`/chat/completions`](#sample-usage), `/completions`, [`/embeddings`](#embedding-models), [`/audio/speech`](#text-to-speech-apis), [`/fine_tuning`](#fine-tuning-apis), [`/batches`](#batch-apis), [`/files`](#batch-apis), [`/images`](#image-generation-models) |
| Supported Operations | [`/chat/completions`](#sample-usage), `/completions`, [`/embeddings`](#embedding-models), [`/audio/speech`](#text-to-speech-apis), [`/fine_tuning`](#fine-tuning-apis), [`/batches`](#batch-apis), [`/files`](#batch-apis), [`/images`](#image-generation-models), [`/rerank`](#rerank-api) |
<br />
@ -1604,6 +1604,53 @@ litellm.vertex_location = "us-central1 # Your Location
| gemini-2.5-flash-preview-09-2025 | `completion('gemini-2.5-flash-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-preview-09-2025', messages)` |
| gemini-2.5-flash-lite-preview-09-2025 | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` |
## Private Service Connect (PSC) Endpoints
LiteLLM supports Vertex AI models deployed to Private Service Connect (PSC) endpoints, allowing you to use custom `api_base` URLs for private deployments.
### Usage
```python
from litellm import completion
# Use PSC endpoint with custom api_base
response = completion(
model="vertex_ai/1234567890", # Numeric endpoint ID
messages=[{"role": "user", "content": "Hello!"}],
api_base="http://10.96.32.8", # Your PSC endpoint
vertex_project="my-project-id",
vertex_location="us-central1"
)
```
**Key Features:**
- Supports both numeric endpoint IDs and custom model names
- Works with both completion and embedding endpoints
- Automatically constructs full PSC URL: `{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}`
- Compatible with streaming requests
### Configuration
Add PSC endpoints to your `config.yaml`:
```yaml
model_list:
- model_name: psc-gemini
litellm_params:
model: vertex_ai/1234567890 # Numeric endpoint ID
api_base: "http://10.96.32.8" # Your PSC endpoint
vertex_project: "my-project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json"
- model_name: psc-embedding
litellm_params:
model: vertex_ai/text-embedding-004
api_base: "http://10.96.32.8" # Your PSC endpoint
vertex_project: "my-project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json"
```
## Fine-tuned Models
You can call fine-tuned Vertex AI Gemini models through LiteLLM
@ -2042,515 +2089,6 @@ curl http://0.0.0.0:4000/v1/chat/completions \
| code-gecko@latest| `completion('code-gecko@latest', messages)` |
## **Embedding Models**
#### Usage - Embedding
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
from litellm import embedding
litellm.vertex_project = "hardy-device-38811" # Your Project ID
litellm.vertex_location = "us-central1" # proj location
response = embedding(
model="vertex_ai/textembedding-gecko",
input=["good morning from litellm"],
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
1. Add model to config.yaml
```yaml
model_list:
- model_name: snowflake-arctic-embed-m-long-1731622468876
litellm_params:
model: vertex_ai/<your-model-id>
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request using OpenAI Python SDK, Langchain Python SDK
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="snowflake-arctic-embed-m-long-1731622468876",
input = ["good morning from litellm", "this is another item"],
)
print(response)
```
</TabItem>
</Tabs>
#### Supported Embedding Models
All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| text-embedding-004 | `embedding(model="vertex_ai/text-embedding-004", input)` |
| text-multilingual-embedding-002 | `embedding(model="vertex_ai/text-multilingual-embedding-002", input)` |
| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` |
| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` |
| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` |
| textembedding-gecko@001 | `embedding(model="vertex_ai/textembedding-gecko@001", input)` |
| textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` |
| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` |
| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` |
| Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/<your-model-id>", input)` |
### Supported OpenAI (Unified) Params
| [param](../embedding/supported_embedding.md#input-params-for-litellmembedding) | type | [vertex equivalent](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api) |
|-------|-------------|--------------------|
| `input` | **string or List[string]** | `instances` |
| `dimensions` | **int** | `output_dimensionality` |
| `input_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | `task_type` |
#### Usage with OpenAI (Unified) Params
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.embedding(
model="vertex_ai/text-embedding-004",
input=["good morning from litellm", "gm"]
input_type = "RETRIEVAL_DOCUMENT",
dimensions=1,
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="text-embedding-004",
input = ["good morning from litellm", "gm"],
dimensions=1,
extra_body = {
"input_type": "RETRIEVAL_QUERY",
}
)
print(response)
```
</TabItem>
</Tabs>
### Supported Vertex Specific Params
| param | type |
|-------|-------------|
| `auto_truncate` | **bool** |
| `task_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** |
| `title` | **str** |
#### Usage with Vertex Specific Params (Use `task_type` and `title`)
You can pass any vertex specific params to the embedding model. Just pass them to the embedding function like this:
[Relevant Vertex AI doc with all embedding params](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#request_body)
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.embedding(
model="vertex_ai/text-embedding-004",
input=["good morning from litellm", "gm"]
task_type = "RETRIEVAL_DOCUMENT",
title = "test",
dimensions=1,
auto_truncate=True,
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="text-embedding-004",
input = ["good morning from litellm", "gm"],
dimensions=1,
extra_body = {
"task_type": "RETRIEVAL_QUERY",
"auto_truncate": True,
"title": "test",
}
)
print(response)
```
</TabItem>
</Tabs>
## **Multi-Modal Embeddings**
Known Limitations:
- Only supports 1 image / video / image per request
- Only supports GCS or base64 encoded images / videos
### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
Using GCS Images
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input="gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" # will be sent as a gcs image
)
```
Using base 64 encoded images
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input="data:image/jpeg;base64,..." # will be sent as a base64 encoded image
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
1. Add model to config.yaml
```yaml
model_list:
- model_name: multimodalembedding@001
litellm_params:
model: vertex_ai/multimodalembedding@001
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK, Langchain Python SDK
<Tabs>
<TabItem value="OpenAI SDK" label="OpenAI SDK">
Requests with GCS Image / Video URI
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png",
)
print(response)
```
Requests with base64 encoded images
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = "data:image/jpeg;base64,...",
)
print(response)
```
</TabItem>
<TabItem value="langchain" label="Langchain">
Requests with GCS Image / Video URI
```python
from langchain_openai import OpenAIEmbeddings
embeddings_models = "multimodalembedding@001"
embeddings = OpenAIEmbeddings(
model="multimodalembedding@001",
base_url="http://0.0.0.0:4000",
api_key="sk-1234", # type: ignore
)
query_result = embeddings.embed_query(
"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
print(query_result)
```
Requests with base64 encoded images
```python
from langchain_openai import OpenAIEmbeddings
embeddings_models = "multimodalembedding@001"
embeddings = OpenAIEmbeddings(
model="multimodalembedding@001",
base_url="http://0.0.0.0:4000",
api_key="sk-1234", # type: ignore
)
query_result = embeddings.embed_query(
"data:image/jpeg;base64,..."
)
print(query_result)
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="proxy-vtx" label="LiteLLM PROXY (Vertex SDK)">
1. Add model to config.yaml
```yaml
default_vertex_config:
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK
```python
import vertexai
from vertexai.vision_models import Image, MultiModalEmbeddingModel, Video
from vertexai.vision_models import VideoSegmentConfig
from google.auth.credentials import Credentials
LITELLM_PROXY_API_KEY = "sk-1234"
LITELLM_PROXY_BASE = "http://0.0.0.0:4000/vertex-ai"
import datetime
class CredentialsWrapper(Credentials):
def __init__(self, token=None):
super().__init__()
self.token = token
self.expiry = None # or set to a future date if needed
def refresh(self, request):
pass
def apply(self, headers, token=None):
headers['Authorization'] = f'Bearer {self.token}'
@property
def expired(self):
return False # Always consider the token as non-expired
@property
def valid(self):
return True # Always consider the credentials as valid
credentials = CredentialsWrapper(token=LITELLM_PROXY_API_KEY)
vertexai.init(
project="adroit-crow-413218",
location="us-central1",
api_endpoint=LITELLM_PROXY_BASE,
credentials = credentials,
api_transport="rest",
)
model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding")
image = Image.load_from_file(
"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
embeddings = model.get_embeddings(
image=image,
contextual_text="Colosseum",
dimension=1408,
)
print(f"Image Embedding: {embeddings.image_embedding}")
print(f"Text Embedding: {embeddings.text_embedding}")
```
</TabItem>
</Tabs>
### Text + Image + Video Embeddings
<Tabs>
<TabItem value="sdk" label="SDK">
Text + Image
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"] # will be sent as a gcs image
)
```
Text + Video
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image
)
```
Image + Video
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
1. Add model to config.yaml
```yaml
model_list:
- model_name: multimodalembedding@001
litellm_params:
model: vertex_ai/multimodalembedding@001
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK, Langchain Python SDK
Text + Image
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"],
)
print(response)
```
Text + Video
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"],
)
print(response)
```
Image + Video
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"],
)
print(response)
```
</TabItem>
</Tabs>
## **Gemini TTS (Text-to-Speech) Audio Output**
:::info
@ -2935,7 +2473,7 @@ finetune_settings:
ft_job = await client.fine_tuning.jobs.create(
model="gemini-1.0-pro-002", # Vertex model you want to fine-tune
training_file="gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl", # file_id from create file response
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm proxy which provider to use
extra_headers={"custom-llm-provider": "vertex_ai"}, # tell litellm proxy which provider to use
)
```
</TabItem>
@ -2946,8 +2484,8 @@ ft_job = await client.fine_tuning.jobs.create(
curl http://localhost:4000/v1/fine_tuning/jobs \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-H "custom-llm-provider: vertex_ai" \
-d '{
"custom_llm_provider": "vertex_ai",
"model": "gemini-1.0-pro-002",
"training_file": "gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl"
}'
@ -2975,9 +2513,7 @@ ft_job = client.fine_tuning.jobs.create(
"learning_rate_multiplier": 0.1, # learning_rate_multiplier on Vertex
"adapter_size": "ADAPTER_SIZE_ONE" # type: ignore, vertex specific hyperparameter
},
extra_body={
"custom_llm_provider": "vertex_ai",
},
extra_headers={"custom-llm-provider": "vertex_ai"},
)
```
</TabItem>
@ -2988,8 +2524,8 @@ ft_job = client.fine_tuning.jobs.create(
curl http://localhost:4000/v1/fine_tuning/jobs \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-H "custom-llm-provider: vertex_ai" \
-d '{
"custom_llm_provider": "vertex_ai",
"model": "gemini-1.0-pro-002",
"training_file": "gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl",
"hyperparameters": {
@ -3114,3 +2650,101 @@ Once that's done, when you deploy the new container in the Google Cloud Run serv
s/o @[Darien Kindlund](https://www.linkedin.com/in/kindlund/) for this tutorial
## **Rerank API**
Vertex AI supports reranking through the Discovery Engine API, providing semantic ranking capabilities for document retrieval.
### Setup
Set your Google Cloud project ID:
```bash
export VERTEXAI_PROJECT="your-project-id"
```
### Usage
```python
from litellm import rerank
# Using the latest model (recommended)
response = rerank(
model="vertex_ai/semantic-ranker-default@latest",
query="What is Google Gemini?",
documents=[
"Gemini is a cutting edge large language model created by Google.",
"The Gemini zodiac symbol often depicts two figures standing side-by-side.",
"Gemini is a constellation that can be seen in the night sky."
],
top_n=2,
return_documents=True # Set to False for ID-only responses
)
# Using specific model versions
response_v003 = rerank(
model="vertex_ai/semantic-ranker-default-003",
query="What is Google Gemini?",
documents=documents,
top_n=2
)
print(response.results)
```
### Parameters
| Parameter | Type | Description |
|-----------|------|-------------|
| `model` | string | Model name (e.g., `vertex_ai/semantic-ranker-default@latest`) |
| `query` | string | Search query |
| `documents` | list | Documents to rank |
| `top_n` | int | Number of top results to return |
| `return_documents` | bool | Return full content (True) or IDs only (False) |
### Supported Models
- `semantic-ranker-default@latest`
- `semantic-ranker-fast@latest`
- `semantic-ranker-default-003`
- `semantic-ranker-default-002`
For detailed model specifications, see the [Google Cloud ranking API documentation](https://cloud.google.com/generative-ai-app-builder/docs/ranking#rank_or_rerank_a_set_of_records_according_to_a_query).
### Proxy Usage
Add to your `config.yaml`:
```yaml
model_list:
- model_name: semantic-ranker-default@latest
litellm_params:
model: vertex_ai/semantic-ranker-default@latest
vertex_ai_project: "your-project-id"
vertex_ai_location: "us-central1"
vertex_ai_credentials: "path/to/service-account.json"
```
Start the proxy:
```bash
litellm --config /path/to/config.yaml
```
Test with curl:
```bash
curl http://0.0.0.0:4000/rerank \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "semantic-ranker-default@latest",
"query": "What is Google Gemini?",
"documents": [
"Gemini is a cutting edge large language model created by Google.",
"The Gemini zodiac symbol often depicts two figures standing side-by-side.",
"Gemini is a constellation that can be seen in the night sky."
],
"top_n": 2
}'
```

268
docs/my-website/docs/providers/vertex_ai/videos.md Normal file
View file

@ -0,0 +1,268 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Vertex AI Video Generation (Veo)
LiteLLM supports Vertex AI's Veo video generation models using the unified OpenAI video API surface.
| Property | Details |
|-------|-------|
| Description | Google Cloud Vertex AI Veo video generation models |
| Provider Route on LiteLLM | `vertex_ai/` |
| Supported Models | `veo-2.0-generate-001`, `veo-3.0-generate-preview`, `veo-3.0-fast-generate-preview`, `veo-3.1-generate-preview`, `veo-3.1-fast-generate-preview` |
| Cost Tracking | ✅ Duration-based pricing |
| Logging Support | ✅ Full request/response logging |
| Proxy Server Support | ✅ Full proxy integration with virtual keys |
| Spend Management | ✅ Budget tracking and rate limiting |
| Link to Provider Doc | [Vertex AI Veo Documentation ↗](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo-video-generation) |
## Quick Start
### Required Environment Setup
```python
import json
import os
os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
# Option 1: Point to a service account file
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/service_account.json"
# Option 2: Store the service account JSON directly
with open("/path/to/service_account.json", "r", encoding="utf-8") as f:
os.environ["VERTEXAI_CREDENTIALS"] = f.read()
```
### Basic Usage
```python
from litellm import video_generation, video_status, video_content
import json
import os
import time
with open("/path/to/service_account.json", "r", encoding="utf-8") as f:
vertex_credentials = f.read()
response = video_generation(
model="vertex_ai/veo-3.0-generate-preview",
prompt="A cat playing with a ball of yarn in a sunny garden",
vertex_project="your-gcp-project-id",
vertex_location="us-central1",
vertex_credentials=vertex_credentials,
seconds="8",
size="1280x720",
)
print(f"Video ID: {response.id}")
print(f"Initial Status: {response.status}")
# Poll for completion
while True:
status = video_status(
video_id=response.id,
vertex_project="your-gcp-project-id",
vertex_location="us-central1",
vertex_credentials=vertex_credentials,
)
print(f"Current Status: {status.status}")
if status.status == "completed":
break
if status.status == "failed":
raise RuntimeError("Video generation failed")
time.sleep(10)
# Download the rendered video
video_bytes = video_content(
video_id=response.id,
vertex_project="your-gcp-project-id",
vertex_location="us-central1",
vertex_credentials=vertex_credentials,
)
with open("generated_video.mp4", "wb") as f:
f.write(video_bytes)
```
## Supported Models
| Model Name | Description | Max Duration | Status |
|------------|-------------|--------------|--------|
| veo-2.0-generate-001 | Veo 2.0 video generation | 5 seconds | GA |
| veo-3.0-generate-preview | Veo 3.0 high quality | 8 seconds | Preview |
| veo-3.0-fast-generate-preview | Veo 3.0 fast generation | 8 seconds | Preview |
| veo-3.1-generate-preview | Veo 3.1 high quality | 10 seconds | Preview |
| veo-3.1-fast-generate-preview | Veo 3.1 fast | 10 seconds | Preview |
## Video Generation Parameters
LiteLLM converts OpenAI-style parameters to Veo's API shape automatically:
| OpenAI Parameter | Vertex AI Parameter | Description | Example |
|------------------|---------------------|-------------|---------|
| `prompt` | `instances[].prompt` | Text description of the video | "A cat playing" |
| `size` | `parameters.aspectRatio` | Converted to `16:9` or `9:16` | "1280x720" → `16:9` |
| `seconds` | `parameters.durationSeconds` | Clip length in seconds | "8" → `8` |
| `input_reference` | `instances[].image` | Reference image for animation | `open("image.jpg", "rb")` |
| Provider-specific params | `extra_body` | Forwarded to Vertex API | `{"negativePrompt": "blurry"}` |
### Size to Aspect Ratio Mapping
- `1280x720`, `1920x1080` → `16:9`
- `720x1280`, `1080x1920` → `9:16`
- Unknown sizes default to `16:9`
## Async Usage
```python
from litellm import avideo_generation, avideo_status, avideo_content
import asyncio
import json
with open("/path/to/service_account.json", "r", encoding="utf-8") as f:
vertex_credentials = f.read()
async def workflow():
response = await avideo_generation(
model="vertex_ai/veo-3.1-generate-preview",
prompt="Slow motion water droplets splashing into a pool",
seconds="10",
vertex_project="your-gcp-project-id",
vertex_location="us-central1",
vertex_credentials=vertex_credentials,
)
while True:
status = await avideo_status(
video_id=response.id,
vertex_project="your-gcp-project-id",
vertex_location="us-central1",
vertex_credentials=vertex_credentials,
)
if status.status == "completed":
break
if status.status == "failed":
raise RuntimeError("Video generation failed")
await asyncio.sleep(10)
video_bytes = await avideo_content(
video_id=response.id,
vertex_project="your-gcp-project-id",
vertex_location="us-central1",
vertex_credentials=vertex_credentials,
)
with open("veo_water.mp4", "wb") as f:
f.write(video_bytes)
asyncio.run(workflow())
```
## LiteLLM Proxy Usage
Add Veo models to your `config.yaml`:
```yaml
model_list:
- model_name: veo-3
litellm_params:
model: vertex_ai/veo-3.0-generate-preview
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: os.environ/VERTEXAI_LOCATION
vertex_credentials: os.environ/VERTEXAI_CREDENTIALS
```
Start the proxy and make requests:
<Tabs>
<TabItem value="curl" label="Curl">
```bash
# Step 1: Generate video
curl --location 'http://0.0.0.0:4000/videos' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "veo-3",
"prompt": "Aerial shot over a futuristic city at sunrise",
"seconds": "8"
}'
# Step 2: Poll status
curl --location 'http://localhost:4000/v1/videos/{video_id}' \
--header 'x-litellm-api-key: sk-1234'
# Step 3: Download video
curl --location 'http://localhost:4000/v1/videos/{video_id}/content' \
--header 'x-litellm-api-key: sk-1234' \
--output video.mp4
```
</TabItem>
<TabItem value="python" label="Python SDK">
```python
import litellm
litellm.api_base = "http://0.0.0.0:4000"
litellm.api_key = "sk-1234"
response = litellm.video_generation(
model="veo-3",
prompt="Aerial shot over a futuristic city at sunrise",
)
status = litellm.video_status(video_id=response.id)
while status.status not in ["completed", "failed"]:
status = litellm.video_status(video_id=response.id)
if status.status == "completed":
content = litellm.video_content(video_id=response.id)
with open("veo_city.mp4", "wb") as f:
f.write(content)
```
</TabItem>
</Tabs>
## Cost Tracking
LiteLLM records the duration returned by Veo so you can apply duration-based pricing.
```python
with open("/path/to/service_account.json", "r", encoding="utf-8") as f:
vertex_credentials = f.read()
response = video_generation(
model="vertex_ai/veo-2.0-generate-001",
prompt="Flowers blooming in fast forward",
seconds="5",
vertex_project="your-gcp-project-id",
vertex_location="us-central1",
vertex_credentials=vertex_credentials,
)
print(response.usage) # {"duration_seconds": 5.0}
```
## Troubleshooting
- **`vertex_project is required`**: set `VERTEXAI_PROJECT` env var or pass `vertex_project` in the request.
- **`Permission denied`**: ensure the service account has the `Vertex AI User` role and the correct region enabled.
- **Video stuck in `processing`**: Veo operations are long-running. Continue polling every 10–15 seconds up to ~10 minutes.
## See Also
- [OpenAI Video Generation](../openai/videos.md)
- [Azure Video Generation](../azure/videos.md)
- [Gemini Video Generation](../gemini/videos.md)
- [Video Generation API Reference](/docs/videos)

16
docs/my-website/docs/providers/vertex_batch.md
View file

@ -50,7 +50,7 @@ oai_client = OpenAI(
file_obj = oai_client.files.create(
file=open("batch_requests.jsonl", "rb"),
purpose="batch",
extra_body={"custom_llm_provider": "vertex_ai"}
extra_headers={"custom-llm-provider": "vertex_ai"}
)
print(f"File uploaded with ID: {file_obj.id}")
@ -63,9 +63,9 @@ print(f"File uploaded with ID: {file_obj.id}")
curl --request POST \
--url http://localhost:4000/v1/files \
--header 'Content-Type: multipart/form-data' \
--header 'custom-llm-provider: vertex_ai' \
--form purpose=batch \
--form file=@batch_requests.jsonl \
--form custom_llm_provider=vertex_ai
--form file=@batch_requests.jsonl
```
</TabItem>
@ -100,7 +100,7 @@ create_batch_response = oai_client.batches.create(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=batch_input_file_id, # e.g. "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd"
extra_body={"custom_llm_provider": "vertex_ai"}
extra_headers={"custom-llm-provider": "vertex_ai"}
)
print(f"Batch created with ID: {create_batch_response.id}")
@ -113,11 +113,11 @@ print(f"Batch created with ID: {create_batch_response.id}")
curl --request POST \
--url http://localhost:4000/v1/batches \
--header 'Content-Type: application/json' \
--header 'custom-llm-provider: vertex_ai' \
--data '{
"input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
"custom_llm_provider": "vertex_ai"
"completion_window": "24h"
}'
```
@ -162,7 +162,7 @@ Check the status of your batch job. The batch will progress through states: `val
```python showLineNumbers title="retrieve_batch.py"
retrieved_batch = oai_client.batches.retrieve(
batch_id=create_batch_response.id, # Created batch id, e.g. 7814463557919047680
extra_body={"custom_llm_provider": "vertex_ai"}
extra_headers={"custom-llm-provider": "vertex_ai"}
)
print(f"Batch status: {retrieved_batch.status}")
@ -230,7 +230,7 @@ encoded_file_id = urllib.parse.quote_plus(output_file_id)
# Get file content
file_content = oai_client.files.content(
file_id=encoded_file_id,
extra_body={"custom_llm_provider": "vertex_ai"}
extra_headers={"custom-llm-provider": "vertex_ai"}
)
# Process the results

587
docs/my-website/docs/providers/vertex_embedding.md Normal file
View file

@ -0,0 +1,587 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Vertex AI Embedding
## Usage - Embedding
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
from litellm import embedding
litellm.vertex_project = "hardy-device-38811" # Your Project ID
litellm.vertex_location = "us-central1" # proj location
response = embedding(
model="vertex_ai/textembedding-gecko",
input=["good morning from litellm"],
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
1. Add model to config.yaml
```yaml
model_list:
- model_name: snowflake-arctic-embed-m-long-1731622468876
litellm_params:
model: vertex_ai/<your-model-id>
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request using OpenAI Python SDK, Langchain Python SDK
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="snowflake-arctic-embed-m-long-1731622468876",
input = ["good morning from litellm", "this is another item"],
)
print(response)
```
</TabItem>
</Tabs>
#### Supported Embedding Models
All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| text-embedding-004 | `embedding(model="vertex_ai/text-embedding-004", input)` |
| text-multilingual-embedding-002 | `embedding(model="vertex_ai/text-multilingual-embedding-002", input)` |
| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` |
| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` |
| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` |
| textembedding-gecko@001 | `embedding(model="vertex_ai/textembedding-gecko@001", input)` |
| textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` |
| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` |
| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` |
| Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/<your-model-id>", input)` |
### Supported OpenAI (Unified) Params
| [param](../embedding/supported_embedding.md#input-params-for-litellmembedding) | type | [vertex equivalent](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api) |
|-------|-------------|--------------------|
| `input` | **string or List[string]** | `instances` |
| `dimensions` | **int** | `output_dimensionality` |
| `input_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | `task_type` |
#### Usage with OpenAI (Unified) Params
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.embedding(
model="vertex_ai/text-embedding-004",
input=["good morning from litellm", "gm"]
input_type = "RETRIEVAL_DOCUMENT",
dimensions=1,
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="text-embedding-004",
input = ["good morning from litellm", "gm"],
dimensions=1,
extra_body = {
"input_type": "RETRIEVAL_QUERY",
}
)
print(response)
```
</TabItem>
</Tabs>
### Supported Vertex Specific Params
| param | type |
|-------|-------------|
| `auto_truncate` | **bool** |
| `task_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** |
| `title` | **str** |
#### Usage with Vertex Specific Params (Use `task_type` and `title`)
You can pass any vertex specific params to the embedding model. Just pass them to the embedding function like this:
[Relevant Vertex AI doc with all embedding params](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#request_body)
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.embedding(
model="vertex_ai/text-embedding-004",
input=["good morning from litellm", "gm"]
task_type = "RETRIEVAL_DOCUMENT",
title = "test",
dimensions=1,
auto_truncate=True,
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="text-embedding-004",
input = ["good morning from litellm", "gm"],
dimensions=1,
extra_body = {
"task_type": "RETRIEVAL_QUERY",
"auto_truncate": True,
"title": "test",
}
)
print(response)
```
</TabItem>
</Tabs>
## **BGE Embeddings**
Use BGE (Baidu General Embedding) models deployed on Vertex AI.
### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Using BGE on Vertex AI"
import litellm
response = litellm.embedding(
model="vertex_ai/bge/<your-endpoint-id>",
input=["Hello", "World"],
vertex_project="your-project-id",
vertex_location="your-location"
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
1. Add model to config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: bge-embedding
litellm_params:
model: vertex_ai/bge/<your-endpoint-id>
vertex_project: "your-project-id"
vertex_location: "us-central1"
vertex_credentials: your-credentials.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```bash
$ litellm --config /path/to/config.yaml
```
3. Make Request using OpenAI Python SDK
```python showLineNumbers title="Making requests to BGE"
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="bge-embedding",
input=["good morning from litellm", "this is another item"]
)
print(response)
```
Using a Private Service Connect (PSC) endpoint
```yaml showLineNumbers title="config.yaml (PSC)"
model_list:
- model_name: bge-small-en-v1.5
litellm_params:
model: vertex_ai/bge/1234567890
api_base: http://10.96.32.8 # Your PSC IP
vertex_project: my-project-id #optional
vertex_location: us-central1 #optional
```
</TabItem>
</Tabs>
## **Multi-Modal Embeddings**
Known Limitations:
- Only supports 1 image / video / image per request
- Only supports GCS or base64 encoded images / videos
### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
Using GCS Images
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input="gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" # will be sent as a gcs image
)
```
Using base 64 encoded images
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input="data:image/jpeg;base64,..." # will be sent as a base64 encoded image
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
1. Add model to config.yaml
```yaml
model_list:
- model_name: multimodalembedding@001
litellm_params:
model: vertex_ai/multimodalembedding@001
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK, Langchain Python SDK
<Tabs>
<TabItem value="OpenAI SDK" label="OpenAI SDK">
Requests with GCS Image / Video URI
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png",
)
print(response)
```
Requests with base64 encoded images
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = "data:image/jpeg;base64,...",
)
print(response)
```
</TabItem>
<TabItem value="langchain" label="Langchain">
Requests with GCS Image / Video URI
```python
from langchain_openai import OpenAIEmbeddings
embeddings_models = "multimodalembedding@001"
embeddings = OpenAIEmbeddings(
model="multimodalembedding@001",
base_url="http://0.0.0.0:4000",
api_key="sk-1234", # type: ignore
)
query_result = embeddings.embed_query(
"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
print(query_result)
```
Requests with base64 encoded images
```python
from langchain_openai import OpenAIEmbeddings
embeddings_models = "multimodalembedding@001"
embeddings = OpenAIEmbeddings(
model="multimodalembedding@001",
base_url="http://0.0.0.0:4000",
api_key="sk-1234", # type: ignore
)
query_result = embeddings.embed_query(
"data:image/jpeg;base64,..."
)
print(query_result)
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="proxy-vtx" label="LiteLLM PROXY (Vertex SDK)">
1. Add model to config.yaml
```yaml
default_vertex_config:
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK
```python
import vertexai
from vertexai.vision_models import Image, MultiModalEmbeddingModel, Video
from vertexai.vision_models import VideoSegmentConfig
from google.auth.credentials import Credentials
LITELLM_PROXY_API_KEY = "sk-1234"
LITELLM_PROXY_BASE = "http://0.0.0.0:4000/vertex-ai"
import datetime
class CredentialsWrapper(Credentials):
def __init__(self, token=None):
super().__init__()
self.token = token
self.expiry = None # or set to a future date if needed
def refresh(self, request):
pass
def apply(self, headers, token=None):
headers['Authorization'] = f'Bearer {self.token}'
@property
def expired(self):
return False # Always consider the token as non-expired
@property
def valid(self):
return True # Always consider the credentials as valid
credentials = CredentialsWrapper(token=LITELLM_PROXY_API_KEY)
vertexai.init(
project="adroit-crow-413218",
location="us-central1",
api_endpoint=LITELLM_PROXY_BASE,
credentials = credentials,
api_transport="rest",
)
model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding")
image = Image.load_from_file(
"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
embeddings = model.get_embeddings(
image=image,
contextual_text="Colosseum",
dimension=1408,
)
print(f"Image Embedding: {embeddings.image_embedding}")
print(f"Text Embedding: {embeddings.text_embedding}")
```
</TabItem>
</Tabs>
### Text + Image + Video Embeddings
<Tabs>
<TabItem value="sdk" label="SDK">
Text + Image
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"] # will be sent as a gcs image
)
```
Text + Video
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image
)
```
Image + Video
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
1. Add model to config.yaml
```yaml
model_list:
- model_name: multimodalembedding@001
litellm_params:
model: vertex_ai/multimodalembedding@001
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK, Langchain Python SDK
Text + Image
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"],
)
print(response)
```
Text + Video
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"],
)
print(response)
```
Image + Video
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"],
)
print(response)
```
</TabItem>
</Tabs>

237
docs/my-website/docs/providers/vertex_ocr.md Normal file
View file

@ -0,0 +1,237 @@
# Vertex AI OCR
## Overview
| Property | Details |
|-------|-------|
| Description | Vertex AI OCR provides document intelligence capabilities powered by Mistral, enabling text extraction from PDFs and images |
| Provider Route on LiteLLM | `vertex_ai/` |
| Supported Operations | `/ocr` |
| Link to Provider Doc | [Vertex AI ↗](https://cloud.google.com/vertex-ai)
Extract text from documents and images using Vertex AI's OCR models, powered by Mistral.
## Quick Start
### **LiteLLM SDK**
```python showLineNumbers title="SDK Usage"
import litellm
import os
# Set environment variables
os.environ["VERTEXAI_PROJECT"] = "your-project-id"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
# OCR with PDF URL
response = litellm.ocr(
model="vertex_ai/mistral-ocr-2505",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
}
)
# Access extracted text
for page in response.pages:
print(page.text)
```
### **LiteLLM PROXY**
```yaml showLineNumbers title="proxy_config.yaml"
model_list:
- model_name: vertex-ocr
litellm_params:
model: vertex_ai/mistral-ocr-2505
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: os.environ/VERTEXAI_LOCATION
vertex_credentials: path/to/service-account.json # Optional
model_info:
mode: ocr
```
**Start Proxy**
```bash
litellm --config proxy_config.yaml
```
**Call OCR via Proxy**
```bash showLineNumbers title="cURL Request"
curl -X POST http://localhost:4000/ocr \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-api-key" \
-d '{
"model": "vertex-ocr",
"document": {
"type": "document_url",
"document_url": "https://arxiv.org/pdf/2201.04234"
}
}'
```
## Authentication
Vertex AI OCR supports multiple authentication methods:
### Service Account JSON
```python showLineNumbers title="Service Account Auth"
response = litellm.ocr(
model="vertex_ai/mistral-ocr-2505",
document={"type": "document_url", "document_url": "https://..."},
vertex_project="your-project-id",
vertex_location="us-central1",
vertex_credentials="path/to/service-account.json"
)
```
### Application Default Credentials
```python showLineNumbers title="Default Credentials"
# Relies on GOOGLE_APPLICATION_CREDENTIALS environment variable
response = litellm.ocr(
model="vertex_ai/mistral-ocr-2505",
document={"type": "document_url", "document_url": "https://..."},
vertex_project="your-project-id",
vertex_location="us-central1"
)
```
## Document Types
Vertex AI OCR supports both PDFs and images.
### PDF Documents
```python showLineNumbers title="PDF OCR"
response = litellm.ocr(
model="vertex_ai/mistral-ocr-2505",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
},
vertex_project="your-project-id",
vertex_location="us-central1"
)
```
### Image Documents
```python showLineNumbers title="Image OCR"
response = litellm.ocr(
model="vertex_ai/mistral-ocr-2505",
document={
"type": "image_url",
"image_url": "https://example.com/image.png"
},
vertex_project="your-project-id",
vertex_location="us-central1"
)
```
### Base64 Encoded Documents
```python showLineNumbers title="Base64 PDF"
import base64
# Read and encode PDF
with open("document.pdf", "rb") as f:
pdf_base64 = base64.b64encode(f.read()).decode()
response = litellm.ocr(
model="vertex_ai/mistral-ocr-2505",
document={
"type": "document_url",
"document_url": f"data:application/pdf;base64,{pdf_base64}"
},
vertex_project="your-project-id",
vertex_location="us-central1"
)
```
## Supported Parameters
```python showLineNumbers title="All Parameters"
response = litellm.ocr(
model="vertex_ai/mistral-ocr-2505",
document={ # Required: Document to process
"type": "document_url",
"document_url": "https://..."
},
vertex_project="your-project-id", # Required: GCP project ID
vertex_location="us-central1", # Optional: Defaults to us-central1
vertex_credentials="path/to/key.json", # Optional: Service account key
include_image_base64=True, # Optional: Include base64 images
pages=[0, 1, 2], # Optional: Specific pages to process
image_limit=10 # Optional: Limit number of images
)
```
## Response Format
```python showLineNumbers title="Response Structure"
# Response has the following structure
response.pages # List of pages with extracted text
response.model # Model used
response.object # "ocr"
response.usage_info # Token usage information
# Access page content
for page in response.pages:
print(f"Page {page.page_number}:")
print(page.text)
```
## Async Support
```python showLineNumbers title="Async Usage"
import litellm
response = await litellm.aocr(
model="vertex_ai/mistral-ocr-2505",
document={
"type": "document_url",
"document_url": "https://example.com/document.pdf"
},
vertex_project="your-project-id",
vertex_location="us-central1"
)
```
## Cost Tracking
LiteLLM automatically tracks costs for Vertex AI OCR:
- **Cost per page**: $0.0005 (based on $1.50 per 1,000 pages)
```python showLineNumbers title="View Cost"
response = litellm.ocr(
model="vertex_ai/mistral-ocr-2505",
document={"type": "document_url", "document_url": "https://..."},
vertex_project="your-project-id"
)
# Access cost information
print(f"Cost: ${response._hidden_params.get('response_cost', 0)}")
```
## Important Notes
:::info URL Conversion
Vertex AI OCR endpoints don't have internet access. LiteLLM automatically converts public URLs to base64 data URIs before sending requests to Vertex AI.
:::
:::tip Regional Availability
Mistral OCR is available in multiple regions. Specify `vertex_location` to use a region closer to your data:
- `us-central1` (default)
- `europe-west1`
- `asia-southeast1`
:::
## Supported Models
- `mistral-ocr-2505` - Latest Mistral OCR model on Vertex AI
Use the Vertex AI provider prefix: `vertex_ai/<model-name>`

410
docs/my-website/docs/proxy/access_control.md
View file

@ -1,25 +1,342 @@
import Image from '@theme/IdealImage';
# Role-based Access Controls (RBAC)
Role-based access control (RBAC) is based on Organizations, Teams and Internal User Roles
<Image img={require('../../img/litellm_user_heirarchy.png')} style={{ width: '100%', maxWidth: '4000px' }} />
- `Organizations` are the top-level entities that contain Teams.
- `Team` - A Team is a collection of multiple `Internal Users`
- `Internal Users` - users that can create keys, make LLM API calls, view usage on LiteLLM. Users can be on multiple teams.
- `Roles` define the permissions of an `Internal User`
- `Virtual Keys` - Keys are used for authentication to the LiteLLM API. Keys are tied to a `Internal User` and `Team`
- `Virtual Keys` - Keys are used for authentication to the LiteLLM API. Each key can optionally be associated with a `user_id`, a `team_id`, or both:
- **User-only key**: Has a `user_id` but no `team_id`. Tracked individually, deleted when the user is deleted.
- **Team key (Service Account)**: Has a `team_id` but no `user_id`. Shared by the team, not deleted when users are removed. [Learn more about service account keys](https://docs.litellm.ai/docs/proxy/virtual_keys#service-account-keys).
- **User + Team key**: Has both `user_id` and `team_id`. Belongs to a specific user within a team context.
## Roles
### When to Use Each Key Type
| Role Type | Role Name | Permissions |
|-----------|-----------|-------------|
| **Admin** | `proxy_admin` | Admin over the platform |
| | `proxy_admin_viewer` | Can login, view all keys, view all spend. **Cannot** create keys/delete keys/add new users |
| **Organization** | `org_admin` | Admin over the organization. Can create teams and users within their organization |
| **Internal User** | `internal_user` | Can login, view/create/delete their own keys, view their spend. **Cannot** add new users |
| | `internal_user_viewer` | Can login, view their own keys, view their own spend. **Cannot** create/delete keys, add new users |
| Key Type | Use Case | Spend Tracking | Lifecycle |
|----------|----------|----------------|-----------|
| **User-only** | Personal API keys for individual developers | Tracked to the user | Deleted when user is deleted |
| **Team (Service Account)** | Production apps, CI/CD pipelines, shared services | Tracked to the team only | Persists even when team members leave |
| **User + Team** | User working within a team context | Tracked to both user and team | Deleted when user is deleted |
**Example scenarios:**
- Use **user-only keys** for developers testing locally
- Use **team service account keys** for your production application that shouldn't break when employees leave
- Use **user + team keys** when you want individual accountability within a team budget
---
## User Roles
LiteLLM has two types of roles:
1. **Global Proxy Roles** - Platform-wide roles that apply across all organizations and teams
2. **Organization/Team Specific Roles** - Roles scoped to specific organizations or teams (**Premium Feature**)
### Global Proxy Roles
| Role Name | Permissions |
|-----------|-------------|
| `proxy_admin` | Admin over the entire platform. Full control over all organizations, teams, and users |
| `proxy_admin_viewer` | Can login, view all keys, view all spend across the platform. **Cannot** create keys/delete keys/add new users |
| `internal_user` | Can login, view/create (when allowed by team-specific permissions)/delete their own keys, view their spend. **Cannot** add new users |
| `internal_user_viewer` | ⚠️ **DEPRECATED** - Use team/org specific roles instead. Can login, view their own keys, view their own spend. **Cannot** create/delete keys, add new users |
### Organization/Team Specific Roles
| Role Name | Permissions |
|-----------|-------------|
| `org_admin` | Admin over a specific organization. Can create teams and users within their organization ✨ **Premium Feature** |
| `team_admin` | Admin over a specific team. Can manage team members, update team settings, and create keys for their team. ✨ **Premium Feature** |
## What Can Each Role Do?
Here's what each role can actually do. Think of it like levels of access.
---
## Global Proxy Roles
These roles apply across the entire LiteLLM platform, regardless of organization or team boundaries.
### Proxy Admin - Full Access
The proxy admin controls everything. They're like the owner of the whole platform.
**What they can do:**
- Create and manage all organizations
- Create and manage all teams (across all organizations)
- Create and manage all users
- View all spend and usage across the platform
- Create and delete keys for anyone
- Update team budgets, rate limits, and models
- Manage team members and assign roles
**Who should be a proxy admin:** Only the people running the LiteLLM instance.
---
### Proxy Admin Viewer - Platform-Wide Read Access
The proxy admin viewer can see everything across the platform but cannot make changes.
**What they can do:**
- View all organizations, teams, and users
- View all spend and usage across the platform
- View all API keys
- Login to the admin dashboard
**What they cannot do:**
- Create or delete keys
- Add or remove users
- Modify budgets, rate limits, or settings
- Make any changes to the platform
**Who should be a proxy admin viewer:** Finance teams, auditors, or stakeholders who need platform-wide visibility without modification rights.
---
### Internal User
An internal user can create API keys (when allowed by team-specific permissions) and make calls. They see their own stuff only. They can become a team admin or org admin if they are assigned the respective roles.
**What they can do:**
- Create API keys for themselves
- Delete their own API keys
- View their own spend and usage
- Make API calls using their keys
**Who should be an internal user:** Anyone who needs UI access for team/org specific operations **OR** for developers you plan to give multiple keys to.
---
### Internal User Viewer - Read-Only Access
:::warning DEPRECATED
This role is deprecated in favor of team/org specific roles. Use `org_admin` or `team_admin` roles for better granular control over user permissions within organizations and teams.
:::
An internal user viewer can view their own information but cannot create or delete keys.
**What they can do:**
- View their own API keys
- View their own spend and usage
- Login to see their dashboard
**What they cannot do:**
- Create or delete API keys
- Make changes to any settings
- Create teams or add users
- View other people's information
**Who should be an internal user viewer (deprecated):** Consider using team/org specific roles instead for better access control.
---
## Organization/Team Specific Roles
:::info
Organization/Team specific roles are premium features. You need to be a LiteLLM Enterprise user to use them. [Get a 7 day trial here](https://www.litellm.ai/#trial).
:::
These roles are scoped to specific organizations or teams. Users with these roles can only manage resources within their assigned organization or team.
### Org Admin - Organization Level Access
An org admin manages one or more organizations. They can create teams within their organization but can't touch other organizations.
**What they can do:**
- Create teams within their organization
- Add users to teams in their organization
- View spend for their organization
- Create keys for users in their organization
**What they cannot do:**
- Create or manage other organizations
- Modify org budgets / rate limits
- Modify org allowed models (e.g. adding a proxy-level model to the org)
**Who should be an org admin:** Department leads or managers who need to manage multiple teams.
---
### Team Admin - Team Level Access
✨ **This is a Premium Feature**
A team admin manages a specific team. They're like a team lead who can add people, update settings, but only for their team.
**What they can do:**
- Add or remove team members from their team
- Update team members' budgets and rate limits within the team
- Change team settings (budget, rate limits, models)
- Create and delete keys for team members
- Onboard a [team-BYOK](./team_model_add) model to LiteLLM (e.g. onboarding a team's finetuned model)
- Configure [team member permissions](#team-member-permissions) to control what regular team members can do
**What they cannot do:**
- Create new teams
- Modify team's budget / rate limits
- Add/remove global proxy models to their team
**Who should be a team admin:** Team leads who need to manage their team's API access without bothering IT.
:::info How to create a team admin
You need to be a LiteLLM Enterprise user to assign team admins. [Get a 7 day trial here](https://www.litellm.ai/#trial).
```shell
curl -X POST 'http://0.0.0.0:4000/team/member_add' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{"team_id": "team-123", "member": {"role": "admin", "user_id": "user@company.com"}}'
```
:::
---
## Team Member Permissions
✨ **This is a Premium Feature**
Team member permissions allow you to control what regular team members (with role=`user`) can do with API keys in their team. By default, team members can only view key information, but you can grant them additional permissions to create, update, or delete keys.
### How It Works
- **Applies to**: Team members with role=`user` (not team admins or org admins)
- **Scope**: Permissions only apply to keys belonging to their team
- **Configuration**: Set at the team level using `team_member_permissions`
- **Override**: Team admins and org admins always have full permissions regardless of these settings
### Available Permissions
| Permission | Method | Description |
|-----------|--------|-------------|
| `/key/info` | GET | View information about virtual keys in the team |
| `/key/health` | GET | Check health status of virtual keys in the team |
| `/key/list` | GET | List all virtual keys belonging to the team |
| `/key/generate` | POST | Create new virtual keys for the team |
| `/key/service-account/generate` | POST | Create service account keys (not tied to a specific user) for the team |
| `/key/update` | POST | Modify existing virtual keys in the team |
| `/key/delete` | POST | Delete virtual keys belonging to the team |
| `/key/regenerate` | POST | Regenerate virtual keys in the team |
| `/key/block` | POST | Block virtual keys in the team |
| `/key/unblock` | POST | Unblock virtual keys in the team |
### Default Permissions
By default, team members can only:
- `/key/info` - View key information
- `/key/health` - Check key health
### Common Permission Scenarios
**Read-only access** (default):
```json
["/key/info", "/key/health"]
```
**Allow key creation but not deletion**:
```json
["/key/info", "/key/health", "/key/generate", "/key/update"]
```
**Full key management**:
```json
["/key/info", "/key/health", "/key/generate", "/key/update", "/key/delete", "/key/regenerate", "/key/block", "/key/unblock", "/key/list"]
```
### How to Configure Team Member Permissions
#### View Current Permissions
```shell
curl --location 'http://0.0.0.0:4000/team/permissions_list?team_id=team-123' \
--header 'Authorization: Bearer sk-1234'
```
Expected Response:
```json
{
"team_id": "team-123",
"team_member_permissions": ["/key/info", "/key/health"],
"all_available_permissions": ["/key/generate", "/key/update", "/key/delete", ...]
}
```
#### Update Team Member Permissions
```shell
curl --location 'http://0.0.0.0:4000/team/update' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"team_id": "team-123",
"team_member_permissions": ["/key/info", "/key/health", "/key/generate", "/key/update"]
}'
```
This allows team members to:
- View key information
- Create new keys
- Update existing keys
- But NOT delete keys
### Who Can Configure These Permissions?
- **Proxy Admin**: Can configure permissions for any team
- **Org Admin**: Can configure permissions for teams in their organization
- **Team Admin**: Can configure permissions for their own team
---
## Quick Comparison
Here's the quick version:
### Global Proxy Roles
| Action | Proxy Admin | Proxy Admin Viewer | Internal User | Internal User Viewer ⚠️ (Deprecated) |
|--------|-------------|-------------------|---------------|-------------------------------------|
| Create organizations | ✅ | ❌ | ❌ | ❌ |
| Create teams | ✅ | ❌ | ❌ | ❌ |
| Manage all teams | ✅ | ❌ | ❌ | ❌ |
| Create/delete any keys | ✅ | ❌ | ❌ | ❌ |
| Create/delete own keys | ✅ | ❌ | ✅ | ❌ |
| View all platform spend | ✅ | ✅ | ❌ | ❌ |
| View own spend | ✅ | ✅ | ✅ | ✅ |
| View all keys | ✅ | ✅ | ❌ | ❌ |
| View own keys | ✅ | ✅ | ✅ | ✅ |
| Add/remove users | ✅ | ❌ | ❌ | ❌ |
> **Note:** The `internal_user_viewer` role is deprecated. Use team/org specific roles for better granular access control.
### Organization/Team Specific Roles
| Action | Org Admin | Team Admin |
|--------|-----------|------------|
| Create teams (in their org) | ✅ | ❌ |
| Manage teams in their org | ✅ | ❌ |
| Manage their specific team | ✅ | ✅ |
| Add/remove team members | ✅ (in their org) | ✅ (their team only) |
| Update team budgets | ✅ (in their org) | ✅ (their team only) |
| Create keys for team members | ✅ (in their org) | ✅ (their team only) |
| View organization spend | ✅ (their org) | ❌ |
| View team spend | ✅ (in their org) | ✅ (their team) |
| Create organizations | ❌ | ❌ |
| View all platform spend | ❌ | ❌ |
## Onboarding Organizations
✨ **This is a Premium Feature**
### 1. Creating a new Organization
Any user with role=`proxy_admin` can create a new organization
@ -124,18 +441,79 @@ Expected Response
```
### `Organization Admin` - Add an `Internal User`
### 4. `Organization Admin` - Add a Team Admin
The organization admin will use the virtual key created in [step 2](#2-adding-an-org_admin-to-an-organization) to add an Internal User to the `engineering_team` Team.
✨ **This is a Premium Feature**
- We will assign role=`internal_user` so the user can create Virtual Keys for themselves
The organization admin can now add a team admin who will manage the `engineering_team`.
- We assign role=`admin` to make them a team admin for this specific team
- `team_id` is from [step 3](#3-organization-admin---create-a-team)
```shell
curl -X POST 'http://0.0.0.0:4000/team/member_add' \
-H 'Authorization: Bearer sk-1234' \
-H 'Authorization: Bearer sk-7shH8TGMAofR4zQpAAo6kQ' \
-H 'Content-Type: application/json' \
-d '{"team_id": "01044ee8-441b-45f4-be7d-c70e002722d8", "member": {"role": "internal_user", "user_id": "krrish@berri.ai"}}'
-d '{"team_id": "01044ee8-441b-45f4-be7d-c70e002722d8", "member": {"role": "admin", "user_id": "john@company.com"}}'
```
Now `john@company.com` is a team admin. They can manage the `engineering_team` - add members, update budgets, create keys - but they can't touch other teams.
Create a Virtual Key for the team admin:
```shell
curl --location 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer sk-7shH8TGMAofR4zQpAAo6kQ' \
--header 'Content-Type: application/json' \
--data '{"user_id": "john@company.com"}'
```
Expected Response:
```json
{
"models": [],
"user_id": "john@company.com",
"key": "sk-TeamAdminKey123",
"key_name": "sk-...Key123"
}
```
### 5. `Team Admin` - Add Team Members
Now the team admin can use their key to add team members without needing to ask the org admin.
```shell
curl -X POST 'http://0.0.0.0:4000/team/member_add' \
-H 'Authorization: Bearer sk-TeamAdminKey123' \
-H 'Content-Type: application/json' \
-d '{"team_id": "01044ee8-441b-45f4-be7d-c70e002722d8", "member": {"role": "user", "user_id": "krrish@berri.ai"}}'
```
The team admin can also create keys for their team members:
```shell
curl --location 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer sk-TeamAdminKey123' \
--header 'Content-Type: application/json' \
--data '{
"user_id": "krrish@berri.ai",
"team_id": "01044ee8-441b-45f4-be7d-c70e002722d8"
}'
```
### 6. `Team Admin` - Update Team Settings
The team admin can update team budgets and rate limits:
```shell
curl --location 'http://0.0.0.0:4000/team/update' \
--header 'Authorization: Bearer sk-TeamAdminKey123' \
--header 'Content-Type: application/json' \
--data '{
"team_id": "01044ee8-441b-45f4-be7d-c70e002722d8",
"max_budget": 100,
"rpm_limit": 1000
}'
```

10
docs/my-website/docs/proxy/admin_ui_sso.md
View file

@ -320,6 +320,16 @@ Okta requires the `GENERIC_CLIENT_STATE` parameter:
GENERIC_CLIENT_STATE="random-string" # Required for Okta
```
### Okta PKCE
If your Okta application is configured to require PKCE (Proof Key for Code Exchange), enable it by setting:
```bash
GENERIC_CLIENT_USE_PKCE="true"
```
This is required when your Okta app settings enforce PKCE for enhanced security. LiteLLM will automatically handle PKCE parameter generation and verification during the OAuth flow.
### Common Configuration Issues
#### Missing Protocol in Base URL

15
docs/my-website/docs/proxy/caching.md
View file

@ -1018,6 +1018,21 @@ cache_params:
```
## Provider-Specific Optional Parameters Caching
By default, LiteLLM only includes standard OpenAI parameters in cache keys. However, some providers (like Vertex AI) use additional parameters that affect the output but aren't included in the standard cache key generation.
### Enable Provider-Specific Parameter Caching
Add this setting to your `config.yaml` to include provider-specific optional parameters in cache keys:
```yaml
litellm_settings:
cache: True
cache_params:
type: "redis"
enable_caching_on_provider_specific_optional_params: True # Include provider-specific params in cache keys
```
## Advanced - user api key cache ttl
Configure how long the in-memory cache stores the key object (prevents db requests)

59
docs/my-website/docs/proxy/config_settings.md
View file

@ -101,6 +101,7 @@ general_settings:
disable_retry_on_max_parallel_request_limit_error: boolean # turn off retries when max parallel request limit is reached
disable_reset_budget: boolean # turn off reset budget scheduled task
disable_adding_master_key_hash_to_db: boolean # turn off storing master key hash in db, for spend tracking
disable_responses_id_security: boolean # turn off response ID security checks that prevent users from accessing other users' responses
enable_jwt_auth: boolean # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims
enforce_user_param: boolean # requires all openai endpoint requests to have a 'user' param
allowed_routes: ["route1", "route2"] # list of allowed proxy API routes - a user can access. (currently JWT-Auth only)
@ -197,6 +198,7 @@ router_settings:
| disable_retry_on_max_parallel_request_limit_error | boolean | If true, turns off retries when max parallel request limit is reached |
| disable_reset_budget | boolean | If true, turns off reset budget scheduled task |
| disable_adding_master_key_hash_to_db | boolean | If true, turns off storing master key hash in db |
| disable_responses_id_security | boolean | If true, disables response ID security checks that prevent users from accessing response IDs from other users. When false (default), response IDs are encrypted with user information to ensure users can only access their own responses. Applies to /v1/responses endpoints |
| enable_jwt_auth | boolean | allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims. [Doc on JWT Tokens](token_auth) |
| enforce_user_param | boolean | If true, requires all OpenAI endpoint requests to have a 'user' param. [Doc on call hooks](call_hooks)|
| allowed_routes | array of strings | List of allowed proxy API routes a user can access [Doc on controlling allowed routes](enterprise#control-available-public-private-routes)|
@ -230,7 +232,7 @@ router_settings:
| max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. |
| proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** |
| proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** |
| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** |
| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 30 seconds** |
| proxy_batch_polling_interval | int | Time (in seconds) to wait before polling a batch, to check if it's completed. **Default is 6000 seconds (1 hour)** |
| alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) |
| custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) |
@ -340,6 +342,7 @@ router_settings:
| router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) |
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Currently supported: 'router_budget_limiting', 'prompt_caching' |
| ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. |
| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) |
### environment variables - Reference
@ -358,6 +361,10 @@ router_settings:
| AIOHTTP_TRUST_ENV | Flag to enable aiohttp trust environment. When this is set to True, aiohttp will respect HTTP(S)_PROXY env vars. **Default is False**
| AIOHTTP_TTL_DNS_CACHE | DNS cache time-to-live for aiohttp in seconds. **Default is 300**
| ALLOWED_EMAIL_DOMAINS | List of email domains allowed for access
| APSCHEDULER_COALESCE | Whether to combine multiple pending executions of a job into one. **Default is False**
| APSCHEDULER_MAX_INSTANCES | Maximum number of concurrent instances of each job. **Default is 1**
| APSCHEDULER_MISFIRE_GRACE_TIME | Grace time in seconds for misfired jobs. **Default is 1**
| APSCHEDULER_REPLACE_EXISTING | Whether to replace existing jobs with the same ID. **Default is False**
| ARIZE_API_KEY | API key for Arize platform integration
| ARIZE_SPACE_KEY | Space key for Arize platform
| ARGILLA_BATCH_SIZE | Batch size for Argilla logging
@ -389,10 +396,11 @@ router_settings:
| AZURE_CERTIFICATE_PASSWORD | Password for Azure OpenAI certificate
| AZURE_CLIENT_ID | Client ID for Azure services
| AZURE_CLIENT_SECRET | Client secret for Azure services
| AZURE_CODE_INTERPRETER_COST_PER_SESSION | Cost per session for Azure Code Interpreter service
| AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS | Input cost per 1K tokens for Azure Computer Use service
| AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS | Output cost per 1K tokens for Azure Computer Use service
| AZURE_DEFAULT_RESPONSES_API_VERSION | Version of the Azure Default Responses API being used. Default is "preview"
| AZURE_DOCUMENT_INTELLIGENCE_API_VERSION | API version for Azure Document Intelligence service
| AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI | Default DPI (dots per inch) setting for Azure Document Intelligence service
| AZURE_TENANT_ID | Tenant ID for Azure Active Directory
| AZURE_USERNAME | Username for Azure services, use in conjunction with AZURE_PASSWORD for azure ad token with basic username/password workflow
| AZURE_PASSWORD | Password for Azure services, use in conjunction with AZURE_USERNAME for azure ad token with basic username/password workflow
@ -423,6 +431,12 @@ router_settings:
| CLOUDZERO_MAX_FETCHED_DATA_RECORDS | Maximum number of data records to fetch from CloudZero
| CLOUDZERO_TIMEZONE | Timezone for date handling (default: UTC)
| CONFIG_FILE_PATH | File path for configuration file
| CYBERARK_ACCOUNT | CyberArk account name for secret management
| CYBERARK_API_BASE | Base URL for CyberArk API
| CYBERARK_API_KEY | API key for CyberArk secret management service
| CYBERARK_CLIENT_CERT | Path to client certificate for CyberArk authentication
| CYBERARK_CLIENT_KEY | Path to client key for CyberArk authentication
| CYBERARK_USERNAME | Username for CyberArk authentication
| CONFIDENT_API_KEY | API key for DeepEval integration
| CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache
| CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service
@ -439,9 +453,15 @@ router_settings:
| DAYS_IN_A_MONTH | Days in a month for calculation purposes. Default is 28
| DAYS_IN_A_WEEK | Days in a week for calculation purposes. Default is 7
| DAYS_IN_A_YEAR | Days in a year for calculation purposes. Default is 365
| DYNAMOAI_API_KEY | API key for DynamoAI Guardrails service
| DYNAMOAI_API_BASE | Base URL for DynamoAI API. Default is https://api.dynamo.ai
| DYNAMOAI_MODEL_ID | Model ID for DynamoAI tracking/logging purposes
| DYNAMOAI_POLICY_IDS | Comma-separated list of DynamoAI policy IDs to apply
| DD_BASE_URL | Base URL for Datadog integration
| DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration
| _DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration
| DD_AGENT_HOST | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API
| DD_AGENT_PORT | Port of DataDog agent for log intake. Default is 10518
| DD_API_KEY | API key for Datadog integration
| DD_SITE | Site URL for Datadog (e.g., datadoghq.com)
| DD_SOURCE | Source identifier for Datadog logs
@ -456,9 +476,11 @@ router_settings:
| DEFAULT_CLIENT_DISCONNECT_CHECK_TIMEOUT_SECONDS | Timeout in seconds for checking client disconnection. Default is 1
| DEFAULT_COOLDOWN_TIME_SECONDS | Duration in seconds to cooldown a model after failures. Default is 5
| DEFAULT_CRON_JOB_LOCK_TTL_SECONDS | Time-to-live for cron job locks in seconds. Default is 60 (1 minute)
| DEFAULT_DATAFORSEO_LOCATION_CODE | Default location code for DataForSEO search API. Default is 2250 (France)
| DEFAULT_FAILURE_THRESHOLD_PERCENT | Threshold percentage of failures to cool down a deployment. Default is 0.5 (50%)
| DEFAULT_FLUSH_INTERVAL_SECONDS | Default interval in seconds for flushing operations. Default is 5
| DEFAULT_HEALTH_CHECK_INTERVAL | Default interval in seconds for health checks. Default is 300 (5 minutes)
| DEFAULT_HEALTH_CHECK_PROMPT | Default prompt used during health checks for non-image models. Default is "test from litellm"
| DEFAULT_IMAGE_HEIGHT | Default height for images. Default is 300
| DEFAULT_IMAGE_TOKEN_COUNT | Default token count for images. Default is 250
| DEFAULT_IMAGE_WIDTH | Default width for images. Default is 300
@ -470,6 +492,7 @@ router_settings:
| DEFAULT_MAX_RETRIES | Default maximum retry attempts. Default is 2
| DEFAULT_MAX_TOKENS | Default maximum tokens for LLM calls. Default is 4096
| DEFAULT_MAX_TOKENS_FOR_TRITON | Default maximum tokens for Triton models. Default is 2000
| DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE | Default maximum size for redis batch cache. Default is 1000
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
@ -484,6 +507,7 @@ router_settings:
| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH | Default minimal reasoning effort thinking budget for Gemini 2.5 Flash. Default is 512
| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE | Default minimal reasoning effort thinking budget for Gemini 2.5 Flash Lite. Default is 512
| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO | Default minimal reasoning effort thinking budget for Gemini 2.5 Pro. Default is 512
| DEFAULT_REDIS_MAJOR_VERSION | Default Redis major version to assume when version cannot be determined. Default is 7
| DEFAULT_REDIS_SYNC_INTERVAL | Default Redis synchronization interval in seconds. Default is 1
| DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND | Default price per second for Replicate GPU. Default is 0.001400
| DEFAULT_REPLICATE_POLLING_DELAY_SECONDS | Default delay in seconds for Replicate polling. Default is 1
@ -495,11 +519,13 @@ router_settings:
| DEFAULT_SLACK_ALERTING_THRESHOLD | Default threshold for Slack alerting. Default is 300
| DEFAULT_SOFT_BUDGET | Default soft budget for LiteLLM proxy keys. Default is 50.0
| DEFAULT_TRIM_RATIO | Default ratio of tokens to trim from prompt end. Default is 0.75
| DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS | Default duration for video generation in seconds in google. Default is 8
| DIRECT_URL | Direct URL for service endpoint
| DISABLE_ADMIN_UI | Toggle to disable the admin UI
| DISABLE_AIOHTTP_TRANSPORT | Flag to disable aiohttp transport. When this is set to True, litellm will use httpx instead of aiohttp. **Default is False**
| DISABLE_AIOHTTP_TRUST_ENV | Flag to disable aiohttp trust environment. When this is set to True, litellm will not trust the environment for aiohttp eg. `HTTP_PROXY` and `HTTPS_PROXY` environment variables will not be used when this is set to True. **Default is False**
| DISABLE_SCHEMA_UPDATE | Toggle to disable schema updates
| DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE | Threshold for deployment failures per minute before enforcing rate limits in parallel request limiter. Default is 1
| DOCS_DESCRIPTION | Description text for documentation pages
| DOCS_FILTERED | Flag indicating filtered documentation
| DOCS_TITLE | Title of the documentation pages
@ -533,6 +559,7 @@ router_settings:
| GENERIC_CLIENT_ID | Client ID for generic OAuth providers
| GENERIC_CLIENT_SECRET | Client secret for generic OAuth providers
| GENERIC_CLIENT_STATE | State parameter for generic client authentication
| GENERIC_CLIENT_USE_PKCE | Enable PKCE (Proof Key for Code Exchange) for generic OAuth providers. Set to "true" when your OAuth provider requires PKCE. **Default is false**
| GENERIC_SSO_HEADERS | Comma-separated list of additional headers to add to the request - e.g. Authorization=Bearer `<token>`, Content-Type=application/json, etc.
| GENERIC_INCLUDE_CLIENT_ID | Include client ID in requests for OAuth
| GENERIC_SCOPE | Scope settings for generic OAuth providers
@ -555,6 +582,8 @@ router_settings:
| GITHUB_COPILOT_ACCESS_TOKEN_FILE | File to store GitHub Copilot access token for `github_copilot` llm provider
| GREENSCALE_API_KEY | API key for Greenscale service
| GREENSCALE_ENDPOINT | Endpoint URL for Greenscale service
| GRAYSWAN_API_BASE | Base URL for GraySwan API. Default is https://api.grayswan.ai
| GRAYSWAN_API_KEY | API key for GraySwan Cygnal service
| GOOGLE_APPLICATION_CREDENTIALS | Path to Google Cloud credentials JSON file
| GOOGLE_CLIENT_ID | Client ID for Google OAuth
| GOOGLE_CLIENT_SECRET | Client secret for Google OAuth
@ -565,9 +594,14 @@ router_settings:
| HEROKU_API_KEY | API key for Heroku services
| HF_API_BASE | Base URL for Hugging Face API
| HCP_VAULT_ADDR | Address for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_APPROLE_MOUNT_PATH | Mount path for AppRole authentication in [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault). Default is "approle"
| HCP_VAULT_APPROLE_ROLE_ID | Role ID for AppRole authentication in [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_APPROLE_SECRET_ID | Secret ID for AppRole authentication in [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_CLIENT_CERT | Path to client certificate for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_CLIENT_KEY | Path to client key for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_MOUNT_NAME | Mount name for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_NAMESPACE | Namespace for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_PATH_PREFIX | Path prefix for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_TOKEN | Token for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_CERT_ROLE | Role for [Hashicorp Vault Secret Manager Auth](../secret.md#hashicorp-vault)
| HELICONE_API_KEY | API key for Helicone service
@ -578,6 +612,8 @@ router_settings:
| HUGGINGFACE_API_KEY | API key for Hugging Face API
| HUMANLOOP_PROMPT_CACHE_TTL_SECONDS | Time-to-live in seconds for cached prompts in Humanloop. Default is 60
| IAM_TOKEN_DB_AUTH | IAM token for database authentication
| IBM_GUARDRAILS_API_BASE | Base URL for IBM Guardrails API
| IBM_GUARDRAILS_AUTH_TOKEN | Authorization bearer token for IBM Guardrails API
| INITIAL_RETRY_DELAY | Initial delay in seconds for retrying requests. Default is 0.5
| JITTER | Jitter factor for retry delay calculations. Default is 0.75
| JSON_LOGS | Enable JSON formatted logging
@ -625,6 +661,7 @@ router_settings:
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
| LITELLM_LOG | Enable detailed logging for LiteLLM
| LITELLM_MODEL_COST_MAP_URL | URL for fetching model cost map data. Default is https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json
| LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file
| LITELLM_LOGGER_NAME | Name for OTEL logger
| LITELLM_METER_NAME | Name for OTEL Meter
@ -632,6 +669,7 @@ router_settings:
| LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL
| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
| LITELLM_SALT_KEY | Salt key for encryption in LiteLLM
| LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections.
@ -655,7 +693,7 @@ router_settings:
| MAX_TOKEN_TRIMMING_ATTEMPTS | Maximum number of attempts to trim a token message. Default is 10
| MAXIMUM_TRACEBACK_LINES_TO_LOG | Maximum number of lines to log in traceback in LiteLLM Logs UI. Default is 100
| MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0
| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 20. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 50. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
| MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001
| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
| MISTRAL_API_BASE | Base URL for Mistral API. Default is https://api.mistral.ai
@ -711,10 +749,11 @@ router_settings:
| PROMPTLAYER_API_KEY | API key for PromptLayer integration
| PROXY_ADMIN_ID | Admin identifier for proxy server
| PROXY_BASE_URL | Base URL for proxy service
| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10
| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 30
| PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour)
| PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605
| PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597
| PYTHON_GC_THRESHOLD | GC thresholds ('gen0,gen1,gen2', e.g. '1000,50,50'); defaults to Python’s values.
| PROXY_LOGOUT_URL | URL for logging out of the proxy service
| QDRANT_API_BASE | Base URL for Qdrant API
| QDRANT_API_KEY | API key for Qdrant service
@ -730,14 +769,20 @@ router_settings:
| REDIS_GCP_SSL_CA_CERTS | Path to SSL CA certificate file for secure GCP Memorystore Redis connections
| REDOC_URL | The path to the Redoc Fast API documentation. **By default this is "/redoc"**
| REPEATED_STREAMING_CHUNK_LIMIT | Limit for repeated streaming chunks to detect looping. Default is 100
| REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES | Maximum size in bytes for WebSocket messages in realtime connections. Default is None.
| REPLICATE_MODEL_NAME_WITH_ID_LENGTH | Length of Replicate model names with ID. Default is 64
| REPLICATE_POLLING_DELAY_SECONDS | Delay in seconds for Replicate polling operations. Default is 0.5
| REQUEST_TIMEOUT | Timeout in seconds for requests. Default is 6000
| ROUTER_MAX_FALLBACKS | Maximum number of fallbacks for router. Default is 5
| RUNWAYML_DEFAULT_API_VERSION | Default API version for RunwayML service. Default is "2024-11-06"
| RUNWAYML_POLLING_TIMEOUT | Timeout in seconds for RunwayML image generation polling. Default is 600 (10 minutes)
| SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours)
| SEPARATE_HEALTH_APP | If set to '1', runs health endpoints on a separate ASGI app and port. Default: '0'.
| SEPARATE_HEALTH_PORT | Port for the separate health endpoints app. Only used if SEPARATE_HEALTH_APP=1. Default: 4001.
| SERVER_ROOT_PATH | Root path for the server application
| SEND_USER_API_KEY_ALIAS | Flag to send user API key alias to Zscaler AI Guard. Default is False
| SEND_USER_API_KEY_TEAM_ID | Flag to send user API key team ID to Zscaler AI Guard. Default is False
| SEND_USER_API_KEY_USER_ID | Flag to send user API key user ID to Zscaler AI Guard. Default is False
| SET_VERBOSE | Flag to enable verbose logging
| SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD | Minimum number of requests to consider "reasonable traffic" for single-deployment cooldown logic. Default is 1000
| SLACK_DAILY_REPORT_FREQUENCY | Frequency of daily Slack reports (e.g., daily, weekly)
@ -752,6 +797,7 @@ router_settings:
| SPEND_LOGS_URL | URL for retrieving spend logs
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000
| SSL_CERTIFICATE | Path to the SSL certificate file
| SSL_ECDH_CURVE | ECDH curve for SSL/TLS key exchange (e.g., 'X25519' to disable PQC).
| SSL_SECURITY_LEVEL | [BETA] Security level for SSL/TLS connections. E.g. `DEFAULT@SECLEVEL=1`
| SSL_VERIFY | Flag to enable or disable SSL certificate verification
| SSL_CERT_FILE | Path to the SSL certificate file for custom CA bundle
@ -784,4 +830,7 @@ router_settings:
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000
| COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY | Maximum size for CoroutineChecker in-memory cache. Default is 1000
| DEFAULT_SHARED_HEALTH_CHECK_TTL | Time-to-live in seconds for cached health check results in shared health check mode. Default is 300 (5 minutes)
| DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL | Time-to-live in seconds for health check lock in shared health check mode. Default is 60 (1 minute)
| DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL | Time-to-live in seconds for health check lock in shared health check mode. Default is 60 (1 minute)
| ZSCALER_AI_GUARD_API_KEY | API key for Zscaler AI Guard service
| ZSCALER_AI_GUARD_POLICY_ID | Policy ID for Zscaler AI Guard guardrails
| ZSCALER_AI_GUARD_URL | Base URL for Zscaler AI Guard API. Default is https://api.us1.zseclipse.net/v1/detection/execute-policy

20
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -2,14 +2,14 @@ import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# 💸 Spend Tracking
# Spend Tracking
Track spend for keys, users, and teams across 100+ LLMs.
LiteLLM automatically tracks spend for all known models. See our [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
:::tip Keep Pricing Data Updated
[Sync model pricing data from GitHub](../sync_models_github.md) to ensure accurate cost tracking.
[Sync model pricing data from GitHub](./sync_models_github.md) to ensure accurate cost tracking.
:::
### How to Track Spend with LiteLLM
@ -23,7 +23,7 @@ LiteLLM automatically tracks spend for all known models. See our [model cost map
<Tabs>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
```python
```python title="Send Request with Spend Tracking" showLineNumbers
import openai
client = openai.OpenAI(
api_key="sk-1234",
@ -55,7 +55,7 @@ print(response)
Pass `metadata` as part of the request body
```shell
```shell title="Curl Request with Spend Tracking" showLineNumbers
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
@ -77,7 +77,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
</TabItem>
<TabItem value="langchain" label="Langchain">
```python
```python title="Langchain with Spend Tracking" showLineNumbers
from langchain.chat_models import ChatOpenAI
from langchain.prompts.chat import (
ChatPromptTemplate,
@ -131,7 +131,7 @@ Expect to see `x-litellm-response-cost` in the response headers with calculated
The following spend gets tracked in Table `LiteLLM_SpendLogs`
```json
```json title="Spend Log Entry Format" showLineNumbers
{
"api_key": "fe6b0cab4ff5a5a8df823196cc8a450*****", # Hash of API Key used
"user": "default_user", # Internal User (LiteLLM_UserTable) that owns `api_key=sk-1234`.
@ -169,7 +169,7 @@ Schedule a [meeting with us to get your Enterprise License](https://calendly.com
Create Key with with `permissions={"get_spend_routes": true}`
```shell
```shell title="Generate Key with Spend Route Permissions" showLineNumbers
curl --location 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
@ -216,7 +216,7 @@ curl -X POST \
Assuming you have been issuing keys for end users, and setting their `user_id` on the key, you can check their usage.
```shell title="Total for a user API" showLineNumbers
```shell title="Get User Spend - API Request" showLineNumbers
curl -L -X GET 'http://localhost:4000/user/info?user_id=jane_smith' \
-H 'Authorization: Bearer sk-...'
```
@ -840,14 +840,14 @@ The `/spend/logs` endpoint now supports a `summarize` parameter to control data
**Get individual transaction logs:**
```bash
```bash title="Get Individual Transaction Logs" showLineNumbers
curl -X GET "http://localhost:4000/spend/logs?start_date=2024-01-01&end_date=2024-01-02&summarize=false" \
-H "Authorization: Bearer sk-1234"
```
**Get summarized data (default):**
```bash
```bash title="Get Summarized Spend Data" showLineNumbers
curl -X GET "http://localhost:4000/spend/logs?start_date=2024-01-01&end_date=2024-01-02" \
-H "Authorization: Bearer sk-1234"
```

Some files were not shown because too many files have changed in this diff Show more