scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

Merge branch 'BerriAI:main' into main

Browse source
This commit is contained in:
fzowl 2025-02-03 15:08:10 +01:00 • committed by GitHub
commit 559ea56f77
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
625 changed files with 48139 additions and 9228 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

403
.circleci/config.yml
View file

@ -47,9 +47,9 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.54.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install openai==1.54.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "respx==0.21.1"
pip install fastapi
@ -105,7 +105,7 @@ jobs:
command: |
pwd
ls
python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router and not assistants and not langfuse and not caching and not cache" -n 4
python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -165,9 +165,9 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.54.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install openai==1.54.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "respx==0.21.1"
pip install fastapi
@ -264,9 +264,9 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.54.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install openai==1.54.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "respx==0.21.1"
pip install fastapi
@ -367,7 +367,7 @@ jobs:
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
@ -375,10 +375,10 @@ jobs:
- auth_ui_unit_tests_coverage
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
@ -417,12 +417,11 @@ jobs:
- litellm_router_coverage
litellm_proxy_unit_testing: # Runs all tests with the "proxy", "key", "jwt" filenames
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
@ -459,9 +458,9 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.54.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install openai==1.54.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "respx==0.21.1"
pip install fastapi
@ -491,7 +490,6 @@ jobs:
chmod +x docker/entrypoint.sh
./docker/entrypoint.sh
set -e
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@ -516,10 +514,10 @@ jobs:
- litellm_proxy_unit_tests_coverage
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
@ -618,7 +616,7 @@ jobs:
command: |
mv coverage.xml llm_translation_coverage.xml
mv .coverage llm_translation_coverage
# Store test results
- store_test_results:
path: test-results
@ -662,7 +660,7 @@ jobs:
command: |
mv coverage.xml batches_coverage.xml
mv .coverage batches_coverage
# Store test results
- store_test_results:
path: test-results
@ -671,6 +669,52 @@ jobs:
paths:
- batches_coverage.xml
- batches_coverage
litellm_utils_testing:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "respx==0.21.1"
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install numpydoc
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_utils_coverage.xml
mv .coverage litellm_utils_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_utils_coverage.xml
- litellm_utils_coverage
pass_through_unit_testing:
docker:
- image: cimg/python:3.11
@ -704,7 +748,7 @@ jobs:
command: |
mv coverage.xml pass_through_unit_tests_coverage.xml
mv .coverage pass_through_unit_tests_coverage
# Store test results
- store_test_results:
path: test-results
@ -746,7 +790,7 @@ jobs:
command: |
mv coverage.xml image_gen_coverage.xml
mv .coverage image_gen_coverage
# Store test results
- store_test_results:
path: test-results
@ -792,7 +836,7 @@ jobs:
command: |
mv coverage.xml logging_coverage.xml
mv .coverage logging_coverage
# Store test results
- store_test_results:
path: test-results
@ -824,6 +868,7 @@ jobs:
pip install "boto3==1.34.34"
pip install jinja2
pip install tokenizers=="0.20.0"
pip install uvloop==0.21.0
pip install jsonschema
- run:
name: Run tests
@ -831,7 +876,7 @@ jobs:
pwd
ls
python -m pytest -vv tests/local_testing/test_basic_python_version.py
installing_litellm_on_python_3_13:
docker:
- image: cimg/python:3.13.1
@ -851,12 +896,79 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "tomli==2.2.1"
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/local_testing/test_basic_python_version.py
helm_chart_testing:
machine:
image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker
resource_class: medium
working_directory: ~/project
steps:
- checkout
# Install Helm
- run:
name: Install Helm
command: |
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
# Install kind
- run:
name: Install Kind
command: |
curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
chmod +x ./kind
sudo mv ./kind /usr/local/bin/kind
# Install kubectl
- run:
name: Install kubectl
command: |
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
chmod +x kubectl
sudo mv kubectl /usr/local/bin/
# Create kind cluster
- run:
name: Create Kind Cluster
command: |
kind create cluster --name litellm-test
# Run helm lint
- run:
name: Run helm lint
command: |
helm lint ./deploy/charts/litellm-helm
# Run helm tests
- run:
name: Run helm tests
command: |
helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml
# Wait for pod to be ready
echo "Waiting 30 seconds for pod to be ready..."
sleep 30
# Print pod logs before running tests
echo "Printing pod logs..."
kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
# Run the helm tests
helm test litellm --logs
helm test litellm --logs
# Cleanup
- run:
name: Cleanup
command: |
kind delete cluster --name litellm-test
when: always # This ensures cleanup runs even if previous steps fail
check_code_and_doc_quality:
docker:
@ -875,14 +987,18 @@ jobs:
pip install ruff
pip install pylint
pip install pyright
pip install beautifulsoup4
pip install .
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
- run: python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
- run: ruff check ./litellm
# - run: python ./tests/documentation_tests/test_general_setting_keys.py
- run: python ./tests/code_coverage_tests/router_code_coverage.py
- run: python ./tests/code_coverage_tests/callback_manager_test.py
- run: python ./tests/code_coverage_tests/recursive_detector.py
- run: python ./tests/code_coverage_tests/test_router_strategy_async.py
- run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
- run: python ./tests/code_coverage_tests/bedrock_pricing.py
- run: python ./tests/documentation_tests/test_env_keys.py
- run: python ./tests/documentation_tests/test_router_settings.py
- run: python ./tests/documentation_tests/test_api_docs.py
@ -928,7 +1044,7 @@ jobs:
cat docker_output.log
exit 1
fi
build_and_test:
machine:
image: ubuntu-2204:2023.10.1
@ -974,9 +1090,9 @@ jobs:
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
pip install numpydoc
pip install prisma
pip install fastapi
pip install jsonschema
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "gunicorn==21.2.0"
pip install "anyio==3.7.1"
@ -984,7 +1100,22 @@ jobs:
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.54.0 "
# Run pytest and generate JUnit XML report
- run:
name: Install Grype
command: |
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
- run:
name: Build and Scan Docker Images
command: |
# Build and scan Dockerfile.database
echo "Building and scanning Dockerfile.database..."
docker build -t litellm-database:latest -f ./docker/Dockerfile.database .
grype litellm-database:latest --fail-on high
# Build and scan main Dockerfile
echo "Building and scanning main Dockerfile..."
docker build -t litellm:latest .
grype litellm:latest --fail-on high
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@ -1009,6 +1140,9 @@ jobs:
-e AWS_REGION_NAME=$AWS_REGION_NAME \
-e AUTO_INFER_REGION=True \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
@ -1092,9 +1226,9 @@ jobs:
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
pip install numpydoc
pip install prisma
pip install fastapi
pip install jsonschema
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "gunicorn==21.2.0"
pip install "anyio==3.7.1"
@ -1128,6 +1262,9 @@ jobs:
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
-e AUTO_INFER_REGION=True \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
@ -1211,9 +1348,9 @@ jobs:
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
pip install numpydoc
pip install prisma
pip install fastapi
pip install jsonschema
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "gunicorn==21.2.0"
pip install "anyio==3.7.1"
@ -1244,6 +1381,9 @@ jobs:
-e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
-e APORIA_API_KEY_1=$APORIA_API_KEY_1 \
-e COHERE_API_KEY=$COHERE_API_KEY \
@ -1276,8 +1416,9 @@ jobs:
pwd
ls
python -m pytest -vv tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
# Clean up first container
no_output_timeout:
120m
# Clean up first container
- run:
name: Stop and remove first container
command: |
@ -1323,7 +1464,104 @@ jobs:
# Store test results
- store_test_results:
path: test-results
proxy_build_from_pip_tests:
# Change from docker to machine executor
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
# Remove Docker CLI installation since it's already available in machine executor
- run:
name: Install Python 3.13
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.13 -y
conda activate myenv
python --version
- run:
name: Install Dependencies
command: |
pip install "pytest==7.3.1"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
python -m pip install --upgrade pip
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install mypy
- run:
name: Build Docker image
command: |
cd docker/build_from_pip
docker build -t my-app:latest -f Dockerfile.build_from_pip .
- run:
name: Run Docker container
# intentionally give bad redis credentials here
# the OTEL test - should get this as a trace
command: |
cd docker/build_from_pip
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$PROXY_DATABASE_URL \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e OTEL_EXPORTER="in_memory" \
-e APORIA_API_BASE_2=$APORIA_API_BASE_2 \
-e APORIA_API_KEY_2=$APORIA_API_KEY_2 \
-e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
-e APORIA_API_KEY_1=$APORIA_API_KEY_1 \
-e COHERE_API_KEY=$COHERE_API_KEY \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e GCS_FLUSH_INTERVAL="1" \
--name my-app \
-v $(pwd)/litellm_config.yaml:/app/config.yaml \
my-app:latest \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
- run:
name: Install curl and dockerize
command: |
sudo apt-get update
sudo apt-get install -y curl
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f my-app
background: true
- run:
name: Wait for app to be ready
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Run tests
command: |
python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
no_output_timeout:
120m
# Clean up first container
- run:
name: Stop and remove first container
command: |
docker stop my-app
docker rm my-app
proxy_pass_through_endpoint_tests:
machine:
image: ubuntu-2204:2023.10.1
@ -1365,9 +1603,9 @@ jobs:
pip install mypy
pip install pyarrow
pip install numpydoc
pip install prisma
pip install fastapi
pip install jsonschema
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "anyio==3.7.1"
pip install "asyncio==3.4.3"
@ -1388,6 +1626,9 @@ jobs:
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e GEMINI_API_KEY=$GEMINI_API_KEY \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
@ -1469,7 +1710,6 @@ jobs:
- codecov/upload:
file: ./coverage.xml
publish_to_pypi:
docker:
- image: cimg/python:3.8
@ -1496,7 +1736,6 @@ jobs:
circleci step halt
fi
- run:
name: Checkout code
command: git checkout $CIRCLE_SHA1
@ -1585,9 +1824,9 @@ jobs:
pip install mypy
pip install pyarrow
pip install numpydoc
pip install prisma
pip install fastapi
pip install jsonschema
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "anyio==3.7.1"
pip install "asyncio==3.4.3"
@ -1638,6 +1877,28 @@ jobs:
- store_test_results:
path: test-results
test_nonroot_image:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- run:
name: Build Docker image
command: |
docker build -t non_root_image:latest . -f ./docker/Dockerfile.non_root
- run:
name: Install Container Structure Test
command: |
curl -LO https://github.com/GoogleContainerTools/container-structure-test/releases/download/v1.19.3/container-structure-test-linux-amd64
chmod +x container-structure-test-linux-amd64
sudo mv container-structure-test-linux-amd64 /usr/local/bin/container-structure-test
- run:
name: Run Container Structure Test
command: |
container-structure-test test --image non_root_image:latest --config docker/tests/nonroot.yaml
test_bad_database_url:
machine:
image: ubuntu-2204:2023.10.1
@ -1703,10 +1964,10 @@ workflows:
- /litellm_.*/
- litellm_assistants_api_testing:
filters:
branches:
only:
- main
- /litellm_.*/
branches:
only:
- main
- /litellm_.*/
- litellm_router_testing:
filters:
branches:
@ -1749,6 +2010,12 @@ workflows:
only:
- main
- /litellm_.*/
- proxy_build_from_pip_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_pass_through_endpoint_tests:
filters:
branches:
@ -1767,6 +2034,12 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_utils_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- pass_through_unit_testing:
filters:
branches:
@ -1789,6 +2062,7 @@ workflows:
requires:
- llm_translation_testing
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing
- image_gen_testing
- logging_testing
@ -1817,6 +2091,12 @@ workflows:
only:
- main
- /litellm_.*/
- helm_chart_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- load_testing:
filters:
branches:
@ -1838,6 +2118,7 @@ workflows:
- test_bad_database_url
- llm_translation_testing
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing
- image_gen_testing
- logging_testing
@ -1852,10 +2133,10 @@ workflows:
- installing_litellm_on_python
- installing_litellm_on_python_3_13
- proxy_logging_guardrails_model_info_tests
- proxy_build_from_pip_tests
- proxy_pass_through_endpoint_tests
- check_code_and_doc_quality
filters:
branches:
only:
- main

2
.circleci/requirements.txt
View file

@ -9,3 +9,5 @@ anthropic
orjson==3.9.15
pydantic==2.7.1
google-cloud-aiplatform==1.43.0
fastapi-sso==0.10.0
uvloop==0.21.0

1
.dockerignore
View file

@ -9,3 +9,4 @@ tests
.devcontainer
*.tgz
log.txt
docker/Dockerfile.*

20
.github/workflows/stale.yml vendored Normal file
View file

@ -0,0 +1,20 @@
name: "Stale Issue Management"
on:
schedule:
- cron: '0 0 * * *' # Runs daily at midnight UTC
workflow_dispatch:
jobs:
stale:
runs-on: ubuntu-latest
steps:
- uses: actions/stale@v8
with:
repo-token: "${{ secrets.GITHUB_TOKEN }}"
stale-issue-message: "This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
days-before-stale: 90 # Revert to 60 days
days-before-close: 7 # Revert to 7 days
stale-issue-label: "stale"
operations-per-run: 1000

5
.gitignore vendored
View file

@ -48,7 +48,7 @@ deploy/charts/litellm/charts/*
deploy/charts/*.tgz
litellm/proxy/vertex_key.json
**/.vim/
/node_modules
**/node_modules
kub.yaml
loadtest_kub.yaml
litellm/proxy/_new_secret_config.yaml
@ -68,3 +68,6 @@ litellm/proxy/google-cloud-sdk/*
tests/llm_translation/log.txt
venv/
tests/local_testing/log.txt
.codegpt
litellm/proxy/_new_new_secret_config.yaml

23
Dockerfile
View file

@ -1,18 +1,20 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.13.1-slim
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.13.1-slim
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
# Set the working directory to /app
WORKDIR /app
USER root
# Install build dependencies
RUN apt-get clean && apt-get update && \
apt-get install -y gcc python3-dev && \
rm -rf /var/lib/apt/lists/*
RUN apk update && \
apk add --no-cache gcc python3-dev openssl openssl-dev
RUN pip install --upgrade pip && \
pip install build
@ -49,8 +51,12 @@ RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE AS runtime
# Update dependencies and clean up - handles debian security issue
RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
# Ensure runtime stage runs as root
USER root
# Install runtime dependencies
RUN apk update && \
apk add --no-cache openssl
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -67,10 +73,11 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
# Generate prisma client
RUN prisma generate
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
ENTRYPOINT ["litellm"]
ENTRYPOINT ["docker/prod_entrypoint.sh"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
CMD ["--port", "4000"]

10
README.md
View file

@ -175,12 +175,12 @@ for part in response:
## Logging Observability ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack, MLflow
LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack
```python
from litellm import completion
## set env variables for logging tools
## set env variables for logging tools (when using MLflow, no API key set up is required)
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
os.environ["HELICONE_API_KEY"] = "your-helicone-auth-key"
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
@ -190,7 +190,7 @@ os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
os.environ["OPENAI_API_KEY"]
# set callbacks
litellm.success_callback = ["lunary", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
litellm.success_callback = ["lunary", "mlflow", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
#openai call
response = completion(model="anthropic/claude-3-sonnet-20240229", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
@ -262,7 +262,7 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# Add the litellm salt key - you cannot change this after adding a model
# It is used to encrypt / decrypt your LLM API Key credentials
# We recommned - https://1password.com/password-generator/
# We recommend - https://1password.com/password-generator/
# password generator to get a random hash for litellm salt key
echo 'LITELLM_SALT_KEY="sk-1234"' > .env
@ -358,7 +358,7 @@ poetry install -E extra_proxy -E proxy
Step 3: Test your change:
```
cd litellm/tests # pwd: Documents/litellm/litellm/tests
cd tests # pwd: Documents/litellm/litellm/tests
poetry run flake8
poetry run pytest .
```

4
db_scripts/create_views.py
View file

@ -168,11 +168,11 @@ async def check_view_exists(): # noqa: PLR0915
print("MonthlyGlobalSpendPerUserPerKey Created!") # noqa
try:
await db.query_raw("""SELECT 1 FROM DailyTagSpend LIMIT 1""")
await db.query_raw("""SELECT 1 FROM "DailyTagSpend" LIMIT 1""")
print("DailyTagSpend Exists!") # noqa
except Exception:
sql_query = """
CREATE OR REPLACE VIEW DailyTagSpend AS
CREATE OR REPLACE VIEW "DailyTagSpend" AS
SELECT
jsonb_array_elements_text(request_tags) AS individual_request_tag,
DATE(s."startTime") AS spend_date,

15
deploy/charts/litellm-helm/ci/test-values.yaml Normal file
View file

@ -0,0 +1,15 @@
fullnameOverride: ""
# Disable database deployment and configuration
db:
deployStandalone: false
useExisting: false
# Test environment variables
envVars:
DD_ENV: "dev_helm"
DD_SERVICE: "litellm"
USE_DDTRACE: "true"
# Disable migration job since we're not using a database
migrationJob:
enabled: false

6
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -91,6 +91,12 @@ spec:
name: {{ include "redis.secretName" .Subcharts.redis }}
key: {{include "redis.secretPasswordKey" .Subcharts.redis }}
{{- end }}
{{- if .Values.envVars }}
{{- range $key, $val := .Values.envVars }}
- name: {{ $key }}
value: {{ $val | quote }}
{{- end }}
{{- end }}
envFrom:
{{- range .Values.environmentSecrets }}
- secretRef:

15
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -1,19 +1,27 @@
{{- if .Values.migrationJob.enabled }}
# This job runs the prisma migrations for the LiteLLM DB.
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "litellm.fullname" . }}-migrations
annotations:
argocd.argoproj.io/hook: PreSync
argocd.argoproj.io/hook-delete-policy: Never # keep this resource so we can debug status on ArgoCD
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation # delete old migration on a new deploy in case the migration needs to make updates
checksum/config: {{ toYaml .Values | sha256sum }}
spec:
template:
metadata:
annotations:
{{- with .Values.migrationJob.annotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
containers:
- name: prisma-migrations
image: ghcr.io/berriai/litellm-database:main-latest
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
command: ["python", "litellm/proxy/prisma_migration.py"]
workingDir: "/app"
env:
@ -42,3 +50,4 @@ spec:
value: "false" # always run the migration from the Helm PreSync hook, override the value set
restartPolicy: OnFailure
backoffLimit: {{ .Values.migrationJob.backoffLimit }}
{{- end }}

16
deploy/charts/litellm-helm/templates/tests/test-connection.yaml
View file

@ -10,6 +10,16 @@ spec:
containers:
- name: wget
image: busybox
command: ['wget']
args: ['{{ include "litellm.fullname" . }}:{{ .Values.service.port }}/health/readiness']
restartPolicy: Never
command: ['sh', '-c']
args:
- |
# Wait for a bit to allow the service to be ready
sleep 10
# Try multiple times with a delay between attempts
for i in $(seq 1 30); do
wget -T 5 "{{ include "litellm.fullname" . }}:{{ .Values.service.port }}/health/readiness" && exit 0
echo "Attempt $i failed, waiting..."
sleep 2
done
exit 1
restartPolicy: Never

43
deploy/charts/litellm-helm/templates/tests/test-env-vars.yaml Normal file
View file

@ -0,0 +1,43 @@
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "litellm.fullname" . }}-env-test"
labels:
{{- include "litellm.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: test
image: busybox
command: ['sh', '-c']
args:
- |
# Test DD_ENV
if [ "$DD_ENV" != "dev_helm" ]; then
echo "❌ Environment variable DD_ENV mismatch. Expected: dev_helm, Got: $DD_ENV"
exit 1
fi
echo "✅ Environment variable DD_ENV matches expected value: $DD_ENV"
# Test DD_SERVICE
if [ "$DD_SERVICE" != "litellm" ]; then
echo "❌ Environment variable DD_SERVICE mismatch. Expected: litellm, Got: $DD_SERVICE"
exit 1
fi
echo "✅ Environment variable DD_SERVICE matches expected value: $DD_SERVICE"
# Test USE_DDTRACE
if [ "$USE_DDTRACE" != "true" ]; then
echo "❌ Environment variable USE_DDTRACE mismatch. Expected: true, Got: $USE_DDTRACE"
exit 1
fi
echo "✅ Environment variable USE_DDTRACE matches expected value: $USE_DDTRACE"
env:
- name: DD_ENV
value: {{ .Values.envVars.DD_ENV | quote }}
- name: DD_SERVICE
value: {{ .Values.envVars.DD_SERVICE | quote }}
- name: USE_DDTRACE
value: {{ .Values.envVars.USE_DDTRACE | quote }}
restartPolicy: Never

6
deploy/charts/litellm-helm/values.yaml
View file

@ -186,5 +186,11 @@ migrationJob:
retries: 3 # Number of retries for the Job in case of failure
backoffLimit: 4 # Backoff limit for Job restarts
disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
annotations: {}
# Additional environment variables to be added to the deployment
envVars: {
# USE_DDTRACE: "true"
}

BIN
dist/litellm-1.57.6.tar.gz vendored Normal file
View file

Binary file not shown.

5
docker/Dockerfile.alpine
View file

@ -48,8 +48,11 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
# Set your entrypoint and command
ENTRYPOINT ["litellm"]
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

1
docker/Dockerfile.custom_ui
View file

@ -33,6 +33,7 @@ WORKDIR /app
# Make sure your docker/entrypoint.sh is executable
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
# Expose the necessary port
EXPOSE 4000/tcp

24
docker/Dockerfile.database
View file

@ -1,18 +1,20 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.13.1-slim
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.13.1-slim
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
# Set the working directory to /app
WORKDIR /app
USER root
# Install build dependencies
RUN apt-get clean && apt-get update && \
apt-get install -y gcc python3-dev && \
rm -rf /var/lib/apt/lists/*
RUN apk update && \
apk add --no-cache gcc python3-dev openssl openssl-dev
RUN pip install --upgrade pip && \
pip install build
@ -38,8 +40,12 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE AS runtime
# Update dependencies and clean up - handles debian security issue
RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
# Ensure runtime stage runs as root
USER root
# Install runtime dependencies
RUN apk update && \
apk add --no-cache openssl
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -67,12 +73,12 @@ RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Generate prisma client
RUN prisma generate
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
# # Set your entrypoint and command
ENTRYPOINT ["litellm"]
ENTRYPOINT ["docker/prod_entrypoint.sh"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
# CMD ["--port", "4000", "--detailed_debug"]

51
docker/Dockerfile.non_root
View file

@ -9,13 +9,16 @@ FROM $LITELLM_BUILD_IMAGE AS builder
# Set the working directory to /app
WORKDIR /app
# Set the shell to bash
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
# Install build dependencies
RUN apt-get clean && apt-get update && \
apt-get install -y gcc python3-dev && \
rm -rf /var/lib/apt/lists/*
RUN pip install --upgrade pip && \
pip install build
RUN pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir build
# Copy the current directory contents into the container at /app
COPY . .
@ -39,7 +42,7 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
FROM $LITELLM_RUNTIME_IMAGE AS runtime
# Update dependencies and clean up - handles debian security issue
RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -53,32 +56,42 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# install semantic-cache [Experimental]- we need this here and not in requirements.txt because redisvl pins to pydantic 1.0
RUN pip install redisvl==0.0.7 --no-deps
# install semantic-cache [Experimental]- we need this here and not in requirements.txt because redisvl pins to pydantic 1.0
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT==2.9.0 --no-cache-dir
RUN pip install redisvl==0.0.7 --no-deps --no-cache-dir && \
pip uninstall jwt -y && \
pip uninstall PyJWT -y && \
pip install PyJWT==2.9.0 --no-cache-dir
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Generate prisma client
ENV PRISMA_BINARY_CACHE_DIR=/app/prisma
RUN mkdir -p /.cache
RUN chmod -R 777 /.cache
RUN pip install nodejs-bin
RUN pip install prisma
RUN prisma generate
### Prisma Handling for Non-Root #################################################
# Prisma allows you to specify the binary cache directory to use
ENV PRISMA_BINARY_CACHE_DIR=/nonexistent
RUN pip install --no-cache-dir nodejs-bin prisma
# Make a /non-existent folder and assign chown to nobody
RUN mkdir -p /nonexistent && \
chown -R nobody:nogroup /app && \
chown -R nobody:nogroup /nonexistent && \
chown -R nobody:nogroup /usr/local/lib/python3.13/site-packages/prisma/
RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
# Run Prisma generate as user = nobody
USER nobody
RUN prisma generate
### End of Prisma Handling for Non-Root #########################################
EXPOSE 4000/tcp
# # Set your entrypoint and command
ENTRYPOINT ["docker/prod_entrypoint.sh"]
ENTRYPOINT ["litellm"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
# CMD ["--port", "4000", "--detailed_debug"]
CMD ["--port", "4000"]

23
docker/build_from_pip/Dockerfile.build_from_pip Normal file
View file

@ -0,0 +1,23 @@
FROM cgr.dev/chainguard/python:latest-dev
USER root
WORKDIR /app
ENV HOME=/home/litellm
ENV PATH="${HOME}/venv/bin:$PATH"
# Install runtime dependencies
RUN apk update && \
apk add --no-cache gcc python3-dev openssl openssl-dev
RUN python -m venv ${HOME}/venv
RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip
COPY requirements.txt .
RUN --mount=type=cache,target=${HOME}/.cache/pip \
${HOME}/venv/bin/pip install -r requirements.txt
EXPOSE 4000/tcp
ENTRYPOINT ["litellm"]
CMD ["--port", "4000"]

9
docker/build_from_pip/Readme.md Normal file
View file

@ -0,0 +1,9 @@
# Docker to build LiteLLM Proxy from litellm pip package
### When to use this ?
If you need to build LiteLLM Proxy from litellm pip package, you can use this Dockerfile as a reference.
### Why build from pip package ?
- If your company has a strict requirement around security / building images you can follow steps outlined here

9
docker/build_from_pip/litellm_config.yaml Normal file
View file

@ -0,0 +1,9 @@
model_list:
- model_name: "gpt-4"
litellm_params:
model: openai/fake
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
general_settings:
alerting: ["slack"]

5
docker/build_from_pip/requirements.txt Normal file
View file

@ -0,0 +1,5 @@
litellm[proxy] # Specify the litellm version you want to use
prometheus_client
langfuse
prisma
ddtrace==2.19.0 # for advanced DD tracing / profiling

8
docker/prod_entrypoint.sh Normal file
View file

@ -0,0 +1,8 @@
#!/bin/sh
if [ "$USE_DDTRACE" = "true" ]; then
export DD_TRACE_OPENAI_ENABLED="False"
exec ddtrace-run litellm "$@"
else
exec litellm "$@"
fi

18
docker/tests/nonroot.yaml Normal file
View file

@ -0,0 +1,18 @@
schemaVersion: 2.0.0
metadataTest:
entrypoint: ["docker/prod_entrypoint.sh"]
user: "nobody"
workdir: "/app"
fileExistenceTests:
- name: "Prisma Folder"
path: "/usr/local/lib/python3.13/site-packages/prisma/"
shouldExist: true
uid: 65534
gid: 65534
- name: "Prisma Schema"
path: "/usr/local/lib/python3.13/site-packages/prisma/schema.prisma"
shouldExist: true
uid: 65534
gid: 65534

68
docs/my-website/docs/benchmarks.md
View file

@ -1,21 +1,61 @@
import Image from '@theme/IdealImage';
# Benchmarks
Benchmarks for LiteLLM Gateway (Proxy Server)
Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpoint.
Locust Settings:
- 2500 Users
- 100 user Ramp Up
Use this config for testing:
**Note:** we're currently migrating to aiohttp which has 10x higher throughput. We recommend using the `aiohttp_openai/` provider for load testing.
```yaml
model_list:
- model_name: "fake-openai-endpoint"
litellm_params:
model: aiohttp_openai/any
api_base: https://your-fake-openai-endpoint.com/chat/completions
api_key: "test"
```
### 1 Instance LiteLLM Proxy
In these tests the median latency of directly calling the fake-openai-endpoint is 60ms.
| Metric | Litellm Proxy (1 Instance) |
|--------|------------------------|
| RPS | 475 |
| Median Latency (ms) | 100 |
| Latency overhead added by LiteLLM Proxy | 40ms |
<!-- <Image img={require('../img/1_instance_proxy.png')} /> -->
<!-- ## **Horizontal Scaling - 10K RPS**
<Image img={require('../img/instances_vs_rps.png')} /> -->
#### Key Findings
- Single instance: 475 RPS @ 100ms latency
- 2 LiteLLM instances: 950 RPS @ 100ms latency
- 4 LiteLLM instances: 1900 RPS @ 100ms latency
### 2 Instances
**Adding 1 instance, will double the RPS and maintain the `100ms-110ms` median latency.**
| Metric | Litellm Proxy (2 Instances) |
|--------|------------------------|
| Median Latency (ms) | 100 |
| RPS | 950 |
## Basic Benchmarks
## Machine Spec used for testing
Overhead when using a Deployed Proxy vs Direct to LLM
- Latency overhead added by LiteLLM Proxy: 107ms
Each machine deploying LiteLLM had the following specs:
- 2 CPU
- 4GB RAM
| Metric | Direct to Fake Endpoint | Basic Litellm Proxy |
|--------|------------------------|---------------------|
| RPS | 1196 | 1133.2 |
| Median Latency (ms) | 33 | 140 |
## Logging Callbacks
@ -39,3 +79,9 @@ Using LangSmith has **no impact on latency, RPS compared to Basic Litellm Proxy*
| RPS | 1133.2 | 1135 |
| Median Latency (ms) | 140 | 132 |
## Locust Settings
- 2500 Users
- 100 user Ramp Up

8
docs/my-website/docs/completion/stream.md
View file

@ -3,9 +3,11 @@ import TabItem from '@theme/TabItem';
# Streaming + Async
- [Streaming Responses](#streaming-responses)
- [Async Completion](#async-completion)
- [Async + Streaming Completion](#async-streaming)
| Feature | LiteLLM SDK | LiteLLM Proxy |
|---------|-------------|---------------|
| Streaming | ✅ [start here](#streaming-responses) | ✅ [start here](../proxy/user_keys#streaming) |
| Async | ✅ [start here](#async-completion) | ✅ [start here](../proxy/user_keys#streaming) |
| Async Streaming | ✅ [start here](#async-streaming) | ✅ [start here](../proxy/user_keys#streaming) |
## Streaming Responses
LiteLLM supports streaming the model response back by passing `stream=True` as an argument to the completion function

47
docs/my-website/docs/data_retention.md Normal file
View file

@ -0,0 +1,47 @@
# Data Retention Policy
## LiteLLM Cloud
### Purpose
This policy outlines the requirements and controls/procedures LiteLLM Cloud has implemented to manage the retention and deletion of customer data.
### Policy
For Customers
1. Active Accounts
- Customer data is retained for as long as the customer’s account is in active status. This includes data such as prompts, generated content, logs, and usage metrics.
2. Voluntary Account Closure
- Data enters an “expired” state when the account is voluntarily closed.
- Expired account data will be retained for 30 days (adjust as needed).
- After this period, the account and all related data will be permanently removed from LiteLLM Cloud systems.
- Customers who wish to voluntarily close their account should download or back up their data (manually or via available APIs) before initiating the closure process.
3. Involuntary Suspension
- If a customer account is involuntarily suspended (e.g., due to non-payment or violation of Terms of Service), there is a 14-day (adjust as needed) grace period during which the account will be inaccessible but can be reopened if the customer resolves the issues leading to suspension.
- After the grace period, if the account remains unresolved, it will be closed and the data will enter the “expired” state.
- Once data is in the “expired” state, it will be permanently removed 30 days (adjust as needed) thereafter, unless legal requirements dictate otherwise.
4. Manual Backup of Suspended Accounts
- If a customer wishes to manually back up data contained in a suspended account, they must bring the account back to good standing (by resolving payment or policy violations) to regain interface/API access.
- Data from a suspended account will not be accessible while the account is in suspension status.
- After 14 days of suspension (adjust as needed), if no resolution is reached, the account is closed and data follows the standard “expired” data removal timeline stated above.
5. Custom Retention Policies
- Enterprise customers can configure custom data retention periods based on their specific compliance and business requirements.
- Available customization options include:
- Adjusting the retention period for active data (0-365 days)
- Custom retention policies must be configured through the LiteLLM Cloud dashboard or via API
### Protection of Records
- LiteLLM Cloud takes measures to ensure that all records under its control are protected against loss, destruction, falsification, and unauthorized access or disclosure. These measures are aligned with relevant legislative, regulatory, contractual, and business obligations.
- When working with a third-party CSP, LiteLLM Cloud requests comprehensive information regarding the CSP’s security mechanisms to protect data, including records stored or processed on behalf of LiteLLM Cloud.
- Cloud service providers engaged by LiteLLM Cloud must disclose their safeguarding practices for records they gather and store on LiteLLM Cloud’s behalf.

78
docs/my-website/docs/data_security.md
View file

@ -1,5 +1,25 @@
# Data Privacy and Security
At LiteLLM, **safeguarding your data privacy and security** is our top priority. We recognize the critical importance of the data you share with us and handle it with the highest level of diligence.
With LiteLLM Cloud, we handle:
- Deployment
- Scaling
- Upgrades and security patches
- Ensuring high availability
<iframe
src="https://status.litellm.ai/badge?theme=light"
width="250"
height="30"
className="inline-block dark:hidden"
style={{
colorScheme: "light",
marginTop: "5px",
}}
></iframe>
## Security Measures
### LiteLLM Cloud
@ -12,17 +32,24 @@
- Audit Logs with retention policy
- Control Allowed IP Addresses that can access your Cloud LiteLLM Instance
For security inquiries, please contact us at support@berri.ai
### Self-hosted Instances LiteLLM
- ** No data or telemetry is stored on LiteLLM Servers when you self host **
- For installation and configuration, see: [Self-hosting guided](../docs/proxy/deploy.md)
- **Telemetry** We run no telemetry when you self host LiteLLM
- **No data or telemetry is stored on LiteLLM Servers when you self-host**
- For installation and configuration, see: [Self-hosting guide](../docs/proxy/deploy.md)
- **Telemetry**: We run no telemetry when you self-host LiteLLM
For security inquiries, please contact us at support@berri.ai
## Supported data regions for LiteLLM Cloud
## **Security Certifications**
| **Certification** | **Status** |
|-------------------|-------------------------------------------------------------------------------------------------|
| SOC 2 Type I | Certified. Report available upon request on Enterprise plan. |
| SOC 2 Type II | In progress. Certificate available by April 15th, 2025 |
| ISO27001 | In progress. Certificate available by February 7th, 2025 |
## Supported Data Regions for LiteLLM Cloud
LiteLLM supports the following data regions:
@ -31,7 +58,7 @@ LiteLLM supports the following data regions:
All data, user accounts, and infrastructure are completely separated between these two regions
## Collection of personal data
## Collection of Personal Data
### For Self-hosted LiteLLM Users:
- No personal data is collected or transmitted to LiteLLM servers when you self-host our software.
@ -40,12 +67,13 @@ All data, user accounts, and infrastructure are completely separated between the
### For LiteLLM Cloud Users:
- LiteLLM Cloud tracks LLM usage data - We do not access or store the message / response content of your API requests or responses. You can see the [fields tracked here](https://github.com/BerriAI/litellm/blob/main/schema.prisma#L174)
**How to use and share the personal data**
**How to Use and Share the Personal Data**
- Only proxy admins can view their usage data, and they can only see the usage data of their organization.
- Proxy admins have the ability to invite other users / admins to their server to view their own usage data
- LiteLLM Cloud does not sell or share any usage data with any third parties.
## Cookies information, security and privacy
## Cookies Information, Security, and Privacy
### For Self-hosted LiteLLM Users:
- Cookie data remains within your own infrastructure.
@ -81,6 +109,12 @@ We value the security community's role in protecting our systems and users. To r
We'll review all reports promptly. Note that we don't currently offer a bug bounty program.
## Vulnerability Scanning
- LiteLLM runs [`grype`](https://github.com/anchore/grype) security scans on all built Docker images.
- See [`grype litellm` check on ci/cd](https://github.com/BerriAI/litellm/blob/main/.circleci/config.yml#L1099).
- Current Status: ✅ Passing. 0 High/Critical severity vulnerabilities found.
## Legal/Compliance FAQs
### Procurement Options
@ -89,35 +123,37 @@ We'll review all reports promptly. Note that we don't currently offer a bug boun
2. AWS Marketplace
3. Azure Marketplace
### Vendor Information
Legal Entity Name: Berrie AI Incorporated
Company Phone Number: 7708783106
Number of employees in the company: 2
Number of employees in security team: 2
Point of contact email address for security incidents: krrish@berri.ai
Point of contact email address for general security-related questions: krrish@berri.ai
Has the Vendor been audited / certified? Currently undergoing SOC-2 Certification from Drata
Has the Vendor been audited / certified?
- SOC 2 Type I. Certified. Report available upon request on Enterprise plan.
- SOC 2 Type II. In progress. Certificate available by April 15th, 2025.
- ISO27001. In progress. Certificate available by February 7th, 2025.
Has an information security management system been implemented? Yes - [CodeQL](https://codeql.github.com/)
Has an information security management system been implemented?
- Yes - [CodeQL](https://codeql.github.com/) and a comprehensive ISMS covering multiple security domains.
Is logging of key events - auth, creation, update changes occurring? Yes - we have [audit logs](https://docs.litellm.ai/docs/proxy/multiple_admins#1-switch-on-audit-logs)
Is logging of key events - auth, creation, update changes occurring?
- Yes - we have [audit logs](https://docs.litellm.ai/docs/proxy/multiple_admins#1-switch-on-audit-logs)
Does the Vendor have an established Cybersecurity incident management program? No
Does the Vendor have an established Cybersecurity incident management program?
- Yes, Incident Response Policy available upon request.
Not applicable - LiteLLM is self-hosted, this is the responsibility of the team hosting the proxy. We do provide [alerting](https://docs.litellm.ai/docs/proxy/alerting) and [monitoring](https://docs.litellm.ai/docs/proxy/prometheus) tools to help with this.
Does the vendor have a vulnerability disclosure policy in place? [Yes](https://github.com/BerriAI/litellm?tab=security-ov-file#security-vulnerability-reporting-guidelines)
Does the vendor perform vulnerability scans? No
Does the vendor perform vulnerability scans?
- Yes, regular vulnerability scans are conducted as detailed in the [Vulnerability Scanning](#vulnerability-scanning) section.
Signer Name: Krish Amit Dholakia
Signer Email: krrish@berri.ai
Signer Email: krrish@berri.ai

34
docs/my-website/docs/embedding/supported_embedding.md
View file

@ -323,6 +323,40 @@ response = embedding(
| embed-english-light-v2.0 | `embedding(model="embed-english-light-v2.0", input=["good morning from litellm", "this is another item"])` |
| embed-multilingual-v2.0 | `embedding(model="embed-multilingual-v2.0", input=["good morning from litellm", "this is another item"])` |
## NVIDIA NIM Embedding Models
### API keys
This can be set as env variables or passed as **params to litellm.embedding()**
```python
import os
os.environ["NVIDIA_NIM_API_KEY"] = "" # api key
os.environ["NVIDIA_NIM_API_BASE"] = "" # nim endpoint url
```
### Usage
```python
from litellm import embedding
import os
os.environ['NVIDIA_NIM_API_KEY'] = ""
response = embedding(
model='nvidia_nim/<model_name>',
input=["good morning from litellm"]
)
```
All models listed [here](https://build.nvidia.com/explore/retrieval) are supported:
| Model Name | Function Call |
| :--- | :--- |
| NV-Embed-QA | `embedding(model="nvidia_nim/NV-Embed-QA", input)` |
| nvidia/nv-embed-v1 | `embedding(model="nvidia_nim/nvidia/nv-embed-v1", input)` |
| nvidia/nv-embedqa-mistral-7b-v2 | `embedding(model="nvidia_nim/nvidia/nv-embedqa-mistral-7b-v2", input)` |
| nvidia/nv-embedqa-e5-v5 | `embedding(model="nvidia_nim/nvidia/nv-embedqa-e5-v5", input)` |
| nvidia/embed-qa-4 | `embedding(model="nvidia_nim/nvidia/embed-qa-4", input)` |
| nvidia/llama-3.2-nv-embedqa-1b-v1 | `embedding(model="nvidia_nim/nvidia/llama-3.2-nv-embedqa-1b-v1", input)` |
| nvidia/llama-3.2-nv-embedqa-1b-v2 | `embedding(model="nvidia_nim/nvidia/llama-3.2-nv-embedqa-1b-v2", input)` |
| snowflake/arctic-embed-l | `embedding(model="nvidia_nim/snowflake/arctic-embed-l", input)` |
| baai/bge-m3 | `embedding(model="nvidia_nim/baai/bge-m3", input)` |
## HuggingFace Embedding Models
LiteLLM supports all Feature-Extraction + Sentence Similarity Embedding models: https://huggingface.co/models?pipeline_tag=feature-extraction

56
docs/my-website/docs/enterprise.md
View file

@ -5,63 +5,39 @@ For companies that need SSO, user management and professional support for LiteLL
Get free 7-day trial key [here](https://www.litellm.ai/#trial)
:::
Deploy managed LiteLLM Proxy within your VPC.
Includes all enterprise features.
[**Procurement available via AWS / Azure Marketplace**](./data_security.md#legalcompliance-faqs)
[**Get 7 day trial key**](https://www.litellm.ai/#trial)
This covers:
- **Enterprise Features**
- **Security**
- ✅ [SSO for Admin UI](./proxy/ui#✨-enterprise-features)
- ✅ [Audit Logs with retention policy](./proxy/enterprise#audit-logs)
- ✅ [JWT-Auth](../docs/proxy/token_auth.md)
- ✅ [Control available public, private routes (Restrict certain endpoints on proxy)](./proxy/enterprise#control-available-public-private-routes)
- ✅ [**Secret Managers** AWS Key Manager, Google Secret Manager, Azure Key](./secret)
- ✅ IP address‑based access control lists
- ✅ Track Request IP Address
- ✅ [Use LiteLLM keys/authentication on Pass Through Endpoints](./proxy/pass_through#✨-enterprise---use-litellm-keysauthentication-on-pass-through-endpoints)
- ✅ Set Max Request / File Size on Requests
- ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](./proxy/enterprise#enforce-required-params-for-llm-requests)
- **Customize Logging, Guardrails, Caching per project**
- ✅ [Team Based Logging](./proxy/team_logging.md) - Allow each team to use their own Langfuse Project / custom callbacks
- ✅ [Disable Logging for a Team](./proxy/team_logging.md#disable-logging-for-a-team) - Switch off all logging for a team/project (GDPR Compliance)
- **Controlling Guardrails by Virtual Keys**
- **Spend Tracking, Budgets & Data Exports**
- ✅ [Tracking Spend for Custom Tags](./proxy/enterprise#tracking-spend-for-custom-tags)
- ✅ [Set USD Budgets Spend for Custom Tags](./proxy/provider_budget_routing#-tag-budgets)
- ✅ [Set Model budgets for Virtual Keys](./proxy/users#-virtual-key-model-specific)
- ✅ [Exporting LLM Logs to GCS Bucket, Azure Blob Storage](./proxy/bucket#🪣-logging-gcs-s3-buckets)
- ✅ [API Endpoints to get Spend Reports per Team, API Key, Customer](./proxy/cost_tracking.md#✨-enterprise-api-endpoints-to-get-spend)
- **Prometheus Metrics**
- ✅ [Prometheus Metrics - Num Requests, failures, LLM Provider Outages](./proxy/prometheus)
- ✅ [`x-ratelimit-remaining-requests`, `x-ratelimit-remaining-tokens` for LLM APIs on Prometheus](./proxy/prometheus#✨-enterprise-llm-remaining-requests-and-remaining-tokens)
- **Custom Branding**
- ✅ [Custom Branding + Routes on Swagger Docs](./proxy/enterprise#swagger-docs---custom-routes--branding)
- ✅ [Public Model Hub](../docs/proxy/enterprise.md#public-model-hub)
- ✅ [Custom Email Branding](../docs/proxy/email.md#customizing-email-branding)
- **Guardrails**
- ✅ [Setting team/key based guardrails](./proxy/guardrails/quick_start.md#-control-guardrails-per-project-api-key)
- ✅ [API endpoint listing available guardrails](./proxy/guardrails/bedrock.md#list-guardrails)
- [**Enterprise Features**](./proxy/enterprise)
- ✅ **Feature Prioritization**
- ✅ **Custom Integrations**
- ✅ **Professional Support - Dedicated discord + slack**
Deployment Options:
**Self-Hosted**
1. Manage Yourself - you can deploy our Docker Image or build a custom image from our pip package, and manage your own infrastructure. In this case, we would give you a license key + provide support via a dedicated support channel.
2. We Manage - you give us subscription access on your AWS/Azure/GCP account, and we manage the deployment.
**Managed**
You can use our cloud product where we setup a dedicated instance for you.
## Frequently Asked Questions
### What topics does Professional support cover and what SLAs do you offer?
### SLA's + Professional Support
Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting. We can’t solve your own infrastructure-related issues but we will guide you to fix them.
- 1 hour for Sev0 issues
- 6 hours for Sev1
- 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday)
- 72h SLA for patching vulnerabilities in the software.
**We can offer custom SLAs** based on your needs and the severity of the issue
@ -78,4 +54,8 @@ You just deploy [our docker image](https://docs.litellm.ai/docs/proxy/deploy) an
LITELLM_LICENSE="eyJ..."
```
No data leaves your environment.
No data leaves your environment.
## Data Security / Legal / Compliance FAQs
[Data Security / Legal / Compliance FAQs](./data_security.md)

62
docs/my-website/docs/fine_tuning.md
View file

@ -10,10 +10,12 @@ This is an Enterprise only endpoint [Get Started with Enterprise here](https://c
:::
## Supported Providers
- Azure OpenAI
- OpenAI
- Vertex AI
| Feature | Supported | Notes |
|-------|-------|-------|
| Supported Providers | OpenAI, Azure OpenAI, Vertex AI | - |
| Cost Tracking | 🟡 | [Let us know if you need this](https://github.com/BerriAI/litellm/issues) |
| Logging | ✅ | Works across all logging integrations |
Add `finetune_settings` and `files_settings` to your litellm config.yaml to use the fine-tuning endpoints.
## Example config.yaml for `finetune_settings` and `files_settings`
@ -110,58 +112,6 @@ curl http://localhost:4000/v1/fine_tuning/jobs \
</TabItem>
<TabItem value="Vertex" label="VertexAI">
<Tabs>
<TabItem value="openai" label="OpenAI Python SDK">
```python
ft_job = await client.fine_tuning.jobs.create(
model="gemini-1.0-pro-002", # Vertex model you want to fine-tune
training_file="gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl", # file_id from create file response
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm proxy which provider to use
)
```
</TabItem>
<TabItem value="curl" label="curl (Unified API)">
```shell
curl http://localhost:4000/v1/fine_tuning/jobs \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"custom_llm_provider": "vertex_ai",
"model": "gemini-1.0-pro-002",
"training_file": "gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl"
}'
```
</TabItem>
<TabItem value="curl-vtx" label="curl (VertexAI API)">
:::info
Use this to create Fine tuning Jobs in [the Vertex AI API Format](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/tuning#create-tuning)
:::
```shell
curl http://localhost:4000/v1/projects/tuningJobs \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"baseModel": "gemini-1.0-pro-002",
"supervisedTuningSpec" : {
"training_dataset_uri": "gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl"
}
}'
```
</TabItem>
</Tabs>
</TabItem>
</Tabs>
### Request Body

8
docs/my-website/docs/getting_started.md
View file

@ -80,13 +80,13 @@ except OpenAIError as e:
## Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, Helicone, Promptlayer, Traceloop, Slack
LiteLLM exposes pre defined callbacks to send data to MLflow, Lunary, Langfuse, Helicone, Promptlayer, Traceloop, Slack
```python
from litellm import completion
## set env variables for logging tools
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
## set env variables for logging tools (API key set up is not required when using MLflow)
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings
os.environ["HELICONE_API_KEY"] = "your-helicone-key"
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
os.environ["LANGFUSE_SECRET_KEY"] = ""
@ -94,7 +94,7 @@ os.environ["LANGFUSE_SECRET_KEY"] = ""
os.environ["OPENAI_API_KEY"]
# set callbacks
litellm.success_callback = ["lunary", "langfuse", "helicone"] # log input/output to langfuse, lunary, supabase, helicone
litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to MLflow, langfuse, lunary, helicone
#openai call
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])

31
docs/my-website/docs/image_variations.md Normal file
View file

@ -0,0 +1,31 @@
# [BETA] Image Variations
OpenAI's `/image/variations` endpoint is now supported.
## Quick Start
```python
from litellm import image_variation
import os
# set env vars
os.environ["OPENAI_API_KEY"] = ""
os.environ["TOPAZ_API_KEY"] = ""
# openai call
response = image_variation(
model="dall-e-2", image=image_url
)
# topaz call
response = image_variation(
model="topaz/Standard V2", image=image_url
)
print(response)
```
## Supported Providers
- OpenAI
- Topaz

44
docs/my-website/docs/index.md
View file

@ -108,6 +108,24 @@ response = completion(
</TabItem>
<TabItem value="nvidia" label="NVIDIA">
```python
from litellm import completion
import os
## set ENV variables
os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key"
os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url"
response = completion(
model="nvidia_nim/<model_name>",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="hugging" label="HuggingFace">
```python
@ -274,6 +292,24 @@ response = completion(
</TabItem>
<TabItem value="nvidia" label="NVIDIA">
```python
from litellm import completion
import os
## set ENV variables
os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key"
os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url"
response = completion(
model="nvidia_nim/<model_name>",
messages=[{ "content": "Hello, how are you?","role": "user"}]
stream=True,
)
```
</TabItem>
<TabItem value="hugging" label="HuggingFace">
```python
@ -393,21 +429,21 @@ except OpenAIError as e:
```
### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, Helicone, Promptlayer, Traceloop, Slack
LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, Helicone, Promptlayer, Traceloop, Slack
```python
from litellm import completion
## set env variables for logging tools
## set env variables for logging tools (API key set up is not required when using MLflow)
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings
os.environ["HELICONE_API_KEY"] = "your-helicone-key"
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
os.environ["LANGFUSE_SECRET_KEY"] = ""
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
os.environ["OPENAI_API_KEY"]
# set callbacks
litellm.success_callback = ["lunary", "langfuse", "helicone"] # log input/output to lunary, langfuse, supabase, helicone
litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to lunary, mlflow, langfuse, helicone
#openai call
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])

49
docs/my-website/docs/langchain/langchain.md
View file

@ -111,5 +111,54 @@ chat.invoke(messages)
</TabItem>
</Tabs>
## Use Langchain ChatLiteLLM with MLflow
MLflow provides open-source observability solution for ChatLiteLLM.
To enable the integration, simply call `mlflow.litellm.autolog()` before in your code. No other setup is necessary.
```python
import mlflow
mlflow.litellm.autolog()
```
Once the auto-tracing is enabled, you can invoke `ChatLiteLLM` and see recorded traces in MLflow.
```python
import os
from langchain.chat_models import ChatLiteLLM
os.environ['OPENAI_API_KEY']="sk-..."
chat = ChatLiteLLM(model="gpt-4o-mini")
chat.invoke("Hi!")
```
## Use Langchain ChatLiteLLM with Lunary
```python
import os
from langchain.chat_models import ChatLiteLLM
from langchain.schema import HumanMessage
import litellm
os.environ["LUNARY_PUBLIC_KEY"] = "" # from https://app.lunary.ai/settings
os.environ['OPENAI_API_KEY']="sk-..."
litellm.success_callback = ["lunary"]
litellm.failure_callback = ["lunary"]
chat = ChatLiteLLM(
model="gpt-4o"
messages = [
HumanMessage(
content="what model are you"
)
]
chat(messages)
```
Get more details [here](../observability/lunary_integration.md)
## Use LangChain ChatLiteLLM + Langfuse
Checkout this section [here](../observability/langfuse_integration#use-langchain-chatlitellm--langfuse) for more details on how to integrate Langfuse with ChatLiteLLM.

16
docs/my-website/docs/load_test_advanced.md
View file

@ -25,6 +25,18 @@ Tutorial on how to get to 1K+ RPS with LiteLLM Proxy on locust
callbacks: ["prometheus"] # Enterprise LiteLLM Only - use prometheus to get metrics on your load test
```
**Use this config for testing:**
**Note:** we're currently migrating to aiohttp which has 10x higher throughput. We recommend using the `aiohttp_openai/` provider for load testing.
```yaml
model_list:
- model_name: "fake-openai-endpoint"
litellm_params:
model: aiohttp_openai/any
api_base: https://your-fake-openai-endpoint.com/chat/completions
api_key: "test"
```
## Load Test - Fake OpenAI Endpoint
@ -46,7 +58,7 @@ litellm provides a hosted `fake-openai-endpoint` you can load test against
model_list:
- model_name: fake-openai-endpoint
litellm_params:
model: openai/fake
model: aiohttp_openai/fake
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
@ -170,7 +182,7 @@ Use the following [prometheus metrics to debug your load tests / failures](./pro
## Machine Specifications for Running LiteLLM Proxy
👉 **Number of Replicas of LiteLLM Proxy=20** for getting 1K+ RPS
👉 **Number of Replicas of LiteLLM Proxy=4** for getting 1K+ RPS
| Service | Spec | CPUs | Memory | Architecture | Version|
| --- | --- | --- | --- | --- | --- |

11
docs/my-website/docs/observability/athina_integration.md
View file

@ -79,6 +79,17 @@ Following are the allowed fields in metadata, their types, and their description
* `expected_response: Optional[str]` - This is the reference response to compare against for evaluation purposes. This is useful for segmenting inference calls by expected response.
* `user_query: Optional[str]` - This is the user's query. For conversational applications, this is the user's last message.
## Using a self hosted deployment of Athina
If you are using a self hosted deployment of Athina, you will need to set the `ATHINA_BASE_URL` environment variable to point to your self hosted deployment.
```python
...
os.environ["ATHINA_BASE_URL"]= "http://localhost:9000"
...
```
## Support & Talk with Athina Team
- [Schedule Demo 👋](https://cal.com/shiv-athina/30min)

7
docs/my-website/docs/observability/braintrust.md
View file

@ -67,7 +67,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
}'
```
## Advanced - pass Project ID
## Advanced - pass Project ID or name
<Tabs>
<TabItem value="sdk" label="SDK">
@ -79,7 +79,10 @@ response = litellm.completion(
{"role": "user", "content": "Hi 👋 - i'm openai"}
],
metadata={
"project_id": "my-special-project"
"project_id": "1234",
# passing project_name will try to find a project with that name, or create one if it doesn't exist
# if both project_id and project_name are passed, project_id will be used
# "project_name": "my-special-project"
}
)
```

3
docs/my-website/docs/observability/callbacks.md
View file

@ -7,11 +7,11 @@ liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`,
liteLLM supports:
- [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback)
- [Lunary](https://lunary.ai/docs)
- [Langfuse](https://langfuse.com/docs)
- [LangSmith](https://www.langchain.com/langsmith)
- [Helicone](https://docs.helicone.ai/introduction)
- [Traceloop](https://traceloop.com/docs)
- [Lunary](https://lunary.ai/docs)
- [Athina](https://docs.athina.ai/)
- [Sentry](https://docs.sentry.io/platforms/python/)
- [PostHog](https://posthog.com/docs/libraries/python)
@ -30,6 +30,7 @@ litellm.success_callback=["posthog", "helicone", "langfuse", "lunary", "athina"]
litellm.failure_callback=["sentry", "lunary", "langfuse"]
## set env variables
os.environ['LUNARY_PUBLIC_KEY'] = ""
os.environ['SENTRY_DSN'], os.environ['SENTRY_API_TRACE_RATE']= ""
os.environ['POSTHOG_API_KEY'], os.environ['POSTHOG_API_URL'] = "api-key", "api-url"
os.environ["HELICONE_API_KEY"] = ""

10
docs/my-website/docs/observability/custom_callback.md
View file

@ -20,9 +20,7 @@ class MyCustomHandler(CustomLogger):
def log_post_api_call(self, kwargs, response_obj, start_time, end_time):
print(f"Post-API Call")
def log_stream_event(self, kwargs, response_obj, start_time, end_time):
print(f"On Stream")
def log_success_event(self, kwargs, response_obj, start_time, end_time):
print(f"On Success")
@ -30,9 +28,6 @@ class MyCustomHandler(CustomLogger):
print(f"On Failure")
#### ASYNC #### - for acompletion/aembeddings
async def async_log_stream_event(self, kwargs, response_obj, start_time, end_time):
print(f"On Async Streaming")
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
print(f"On Async Success")
@ -127,8 +122,7 @@ from litellm import acompletion
class MyCustomHandler(CustomLogger):
#### ASYNC ####
async def async_log_stream_event(self, kwargs, response_obj, start_time, end_time):
print(f"On Async Streaming")
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
print(f"On Async Success")

176
docs/my-website/docs/observability/humanloop.md Normal file
View file

@ -0,0 +1,176 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Humanloop
[Humanloop](https://humanloop.com/docs/v5/getting-started/overview) enables product teams to build robust AI features with LLMs, using best-in-class tooling for Evaluation, Prompt Management, and Observability.
## Getting Started
Use Humanloop to manage prompts across all LiteLLM Providers.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import os
import litellm
os.environ["HUMANLOOP_API_KEY"] = "" # [OPTIONAL] set here or in `.completion`
litellm.set_verbose = True # see raw request to provider
resp = litellm.completion(
model="humanloop/gpt-3.5-turbo",
prompt_id="test-chat-prompt",
prompt_variables={"user_message": "this is used"}, # [OPTIONAL]
messages=[{"role": "user", "content": "<IGNORED>"}],
# humanloop_api_key="..." ## alternative to setting env var
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: humanloop/gpt-3.5-turbo
prompt_id: "<humanloop_prompt_id>"
api_key: os.environ/OPENAI_API_KEY
```
2. Start the proxy
```bash
litellm --config config.yaml --detailed_debug
```
3. Test it!
<Tabs>
<TabItem value="curl" label="CURL">
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "THIS WILL BE IGNORED"
}
],
"prompt_variables": {
"key": "this is used"
}
}'
```
</TabItem>
<TabItem value="OpenAI Python SDK" label="OpenAI Python SDK">
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(
model="gpt-3.5-turbo",
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
extra_body={
"prompt_variables": { # [OPTIONAL]
"key": "this is used"
}
}
)
print(response)
```
</TabItem>
</Tabs>
</TabItem>
</Tabs>
**Expected Logs:**
```
POST Request Sent from LiteLLM:
curl -X POST \
https://api.openai.com/v1/ \
-d '{'model': 'gpt-3.5-turbo', 'messages': <YOUR HUMANLOOP PROMPT TEMPLATE>}'
```
## How to set model
## How to set model
### Set the model on LiteLLM
You can do `humanloop/<litellm_model_name>`
<Tabs>
<TabItem value="sdk" label="SDK">
```python
litellm.completion(
model="humanloop/gpt-3.5-turbo", # or `humanloop/anthropic/claude-3-5-sonnet`
...
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: humanloop/gpt-3.5-turbo # OR humanloop/anthropic/claude-3-5-sonnet
prompt_id: <humanloop_prompt_id>
api_key: os.environ/OPENAI_API_KEY
```
</TabItem>
</Tabs>
### Set the model on Humanloop
LiteLLM will call humanloop's `https://api.humanloop.com/v5/prompts/<your-prompt-id>` endpoint, to get the prompt template.
This also returns the template model set on Humanloop.
```bash
{
"template": [
{
... # your prompt template
}
],
"model": "gpt-3.5-turbo" # your template model
}
```

9
docs/my-website/docs/observability/langsmith_integration.md
View file

@ -3,13 +3,6 @@ import Image from '@theme/IdealImage';
# Langsmith - Logging LLM Input/Output
:::tip
This is community maintained, Please make an issue if you run into a bug
https://github.com/BerriAI/litellm
:::
An all-in-one developer platform for every step of the application lifecycle
https://smith.langchain.com/
@ -66,7 +59,7 @@ os.environ["LANGSMITH_API_KEY"] = ""
# LLM API Keys
os.environ['OPENAI_API_KEY']=""
# set langfuse as a callback, litellm will send the data to langfuse
# set langsmith as a callback, litellm will send the data to langsmith
litellm.success_callback = ["langsmith"]
response = litellm.completion(

156
docs/my-website/docs/observability/lunary_integration.md
View file

@ -1,72 +1,78 @@
# Lunary - Logging and tracing LLM input/output
import Image from '@theme/IdealImage';
:::tip
# 🌙 Lunary - GenAI Observability
This is community maintained, Please make an issue if you run into a bug
https://github.com/BerriAI/litellm
[Lunary](https://lunary.ai/) is an open-source platform providing [observability](https://lunary.ai/docs/features/observe), [prompt management](https://lunary.ai/docs/features/prompts), and [analytics](https://lunary.ai/docs/features/observe#analytics) to help team manage and improve LLM chatbots.
:::
[Lunary](https://lunary.ai/) is an open-source AI developer platform providing observability, prompt management, and evaluation tools for AI developers.
You can reach out to us anytime by [email](mailto:hello@lunary.ai) or directly [schedule a Demo](https://lunary.ai/schedule).
<video controls width='900' >
<source src='https://lunary.ai/videos/demo-annotated.mp4'/>
</video>
## Use Lunary to log requests across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM)
liteLLM provides `callbacks`, making it easy for you to log data depending on the status of your responses.
## Usage with LiteLLM Python SDK
### Pre-Requisites
:::info
We want to learn how we can make the callbacks better! Meet the [founders](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) or
join our [discord](https://discord.gg/wuPM9dRgDw)
:::
```shell
pip install litellm lunary
```
### Using Callbacks
### Quick Start
First, sign up to get a public key on the [Lunary dashboard](https://lunary.ai).
First, get your Lunary public key on the [Lunary dashboard](https://app.lunary.ai/).
Use just 2 lines of code, to instantly log your responses **across all providers** with lunary:
Use just 2 lines of code, to instantly log your responses **across all providers** with Lunary:
```python
litellm.success_callback = ["lunary"]
litellm.failure_callback = ["lunary"]
```
Complete code
Complete code:
```python
from litellm import completion
## set env variables
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # from https://app.lunary.ai/)
os.environ["OPENAI_API_KEY"] = ""
# set callbacks
litellm.success_callback = ["lunary"]
litellm.failure_callback = ["lunary"]
#openai call
response = completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}],
model="gpt-4o",
messages=[{"role": "user", "content": "Hi there 👋"}],
user="ishaan_litellm"
)
```
## Templates
### Usage with LangChain ChatLiteLLM
```python
import os
from langchain.chat_models import ChatLiteLLM
from langchain.schema import HumanMessage
import litellm
You can use Lunary to manage prompt templates and use them across all your LLM providers.
os.environ["LUNARY_PUBLIC_KEY"] = "" # from https://app.lunary.ai/settings
os.environ['OPENAI_API_KEY']="sk-..."
Make sure to have `lunary` installed:
litellm.success_callback = ["lunary"]
litellm.failure_callback = ["lunary"]
```bash
pip install lunary
chat = ChatLiteLLM(
model="gpt-4o"
messages = [
HumanMessage(
content="what model are you"
)
]
chat(messages)
```
Then, use the following code to pull templates into Lunary:
### Usage with Prompt Templates
You can use Lunary to manage [prompt templates](https://lunary.ai/docs/features/prompts) and use them across all your LLM providers with LiteLLM.
```python
from litellm import completion
@ -81,9 +87,93 @@ litellm.success_callback = ["lunary"]
result = completion(**template)
```
### Usage with custom chains
You can wrap your LLM calls inside custom chains, so that you can visualize them as traces.
```python
import litellm
from litellm import completion
import lunary
litellm.success_callback = ["lunary"]
litellm.failure_callback = ["lunary"]
@lunary.chain("My custom chain name")
def my_chain(chain_input):
chain_run_id = lunary.run_manager.current_run_id
response = completion(
model="gpt-4o",
messages=[{"role": "user", "content": "Say 1"}],
metadata={"parent_run_id": chain_run_id},
)
response = completion(
model="gpt-4o",
messages=[{"role": "user", "content": "Say 2"}],
metadata={"parent_run_id": chain_run_id},
)
chain_output = response.choices[0].message
return chain_output
my_chain("Chain input")
```
<Image img={require('../../img/lunary-trace.png')} />
## Usage with LiteLLM Proxy Server
### Step1: Install dependencies and set your environment variables
Install the dependencies
```shell
pip install litellm lunary
```
Get you Lunary public key from from https://app.lunary.ai/settings
```shell
export LUNARY_PUBLIC_KEY="<your-public-key>"
```
### Step 2: Create a `config.yaml` and set `lunary` callbacks
```yaml
model_list:
- model_name: "*"
litellm_params:
model: "*"
litellm_settings:
success_callback: ["lunary"]
failure_callback: ["lunary"]
```
### Step 3: Start the LiteLLM proxy
```shell
litellm --config config.yaml
```
### Step 4: Make a request
```shell
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o",
"messages": [
{
"role": "system",
"content": "You are a helpful math tutor. Guide the user through the solution step by step."
},
{
"role": "user",
"content": "how can I solve 8x + 7 = -23"
}
]
}'
```
You can find more details about the different ways of making requests to the LiteLLM proxy on [this page](https://docs.litellm.ai/docs/proxy/user_keys)
## Support & Talk to Founders
- Meet the Lunary team via [email](mailto:hello@lunary.ai).
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬

69
docs/my-website/docs/observability/mlflow.md
View file

@ -1,4 +1,6 @@
# MLflow
import Image from '@theme/IdealImage';
# 🔁 MLflow - OSS LLM Observability and Evaluation
## What is MLflow?
@ -18,7 +20,7 @@ Install MLflow:
pip install mlflow
```
To enable LiteLLM tracing:
To enable MLflow auto tracing for LiteLLM:
```python
import mlflow
@ -29,9 +31,9 @@ mlflow.litellm.autolog()
# litellm.callbacks = ["mlflow"]
```
Since MLflow is open-source, no sign-up or API key is needed to log traces!
Since MLflow is open-source and free, **no sign-up or API key is needed to log traces!**
```
```python
import litellm
import os
@ -53,6 +55,63 @@ Open the MLflow UI and go to the `Traces` tab to view logged traces:
mlflow ui
```
## Tracing Tool Calls
MLflow integration with LiteLLM support tracking tool calls in addition to the messages.
```python
import mlflow
# Enable MLflow auto-tracing for LiteLLM
mlflow.litellm.autolog()
# Define the tool function.
def get_weather(location: str) -> str:
if location == "Tokyo":
return "sunny"
elif location == "Paris":
return "rainy"
return "unknown"
# Define function spec
get_weather_tool = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {
"properties": {
"location": {
"description": "The city and state, e.g., San Francisco, CA",
"type": "string",
},
},
"required": ["location"],
"type": "object",
},
},
}
# Call LiteLLM as usual
response = litellm.completion(
model="gpt-4o-mini",
messages=[
{"role": "user", "content": "What's the weather like in Paris today?"}
],
tools=[get_weather_tool]
)
```
<Image img={require('../../img/mlflow_tool_calling_tracing.png')} />
## Evaluation
MLflow LiteLLM integration allow you to run qualitative assessment against LLM to evaluate or/and monitor your GenAI application.
Visit [Evaluate LLMs Tutorial](../tutorials/eval_suites.md) for the complete guidance on how to run evaluation suite with LiteLLM and MLflow.
## Exporting Traces to OpenTelemetry collectors
MLflow traces are compatible with OpenTelemetry. You can export traces to any OpenTelemetry collector (e.g., Jaeger, Zipkin, Datadog, New Relic) by setting the endpoint URL in the environment variables.
@ -75,7 +134,7 @@ import litellm
import mlflow
from mlflow.entities import SpanType
# Enable LiteLLM tracing
# Enable MLflow auto-tracing for LiteLLM
mlflow.litellm.autolog()

13
docs/my-website/docs/pass_through/intro.md Normal file
View file

@ -0,0 +1,13 @@
# Why Pass-Through Endpoints?
These endpoints are useful for 2 scenarios:
1. **Migrate existing projects** to litellm proxy. E.g: If you have users already in production with Anthropic's SDK, you just need to change the base url to get cost tracking/logging/budgets/etc.
2. **Use provider-specific endpoints** E.g: If you want to use [Vertex AI's token counting endpoint](https://docs.litellm.ai/docs/pass_through/vertex_ai#count-tokens-api)
## How is your request handled?
The request is passed through to the provider's endpoint. The response is then passed back to the client. **No translation is done.**

8
docs/my-website/docs/projects/smolagents.md Normal file
View file

@ -0,0 +1,8 @@
# 🤗 Smolagents
`smolagents` is a barebones library for agents. Agents write python code to call tools and orchestrate other agents.
- [Github](https://github.com/huggingface/smolagents)
- [Docs](https://huggingface.co/docs/smolagents/index)
- [Build your agent](https://huggingface.co/docs/smolagents/guided_tour)

110
docs/my-website/docs/providers/azure.md
View file

@ -10,7 +10,7 @@ import TabItem from '@theme/TabItem';
| Property | Details |
|-------|-------|
| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series |
| Provider Route on LiteLLM | `azure/` |
| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#azure-o-series-models) |
| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/completions`](#azure-instruct-models), [`/embeddings`](../embedding/supported_embedding#azure-openai-embedding-models), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview)
@ -528,6 +528,39 @@ Example video of using `tenant_id`, `client_id`, `client_secret` with LiteLLM Pr
<iframe width="840" height="500" src="https://www.loom.com/embed/70d3f219ee7f4e5d84778b7f17bba506?sid=04b8ff29-485f-4cb8-929e-6b392722f36d" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
### Entrata ID - use client_id, username, password
Here is an example of setting up `client_id`, `azure_username`, `azure_password` in your litellm proxy `config.yaml`
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-v-2
api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
api_version: "2023-05-15"
client_id: os.environ/AZURE_CLIENT_ID
azure_username: os.environ/AZURE_USERNAME
azure_password: os.environ/AZURE_PASSWORD
```
Test it
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data ' {
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}
'
```
### Azure AD Token Refresh - `DefaultAzureCredential`
Use this if you want to use Azure `DefaultAzureCredential` for Authentication on your requests
@ -554,6 +587,16 @@ response = completion(
</TabItem>
<TabItem value="proxy" label="PROXY config.yaml">
1. Add relevant env vars
```bash
export AZURE_TENANT_ID=""
export AZURE_CLIENT_ID=""
export AZURE_CLIENT_SECRET=""
```
2. Setup config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
@ -565,6 +608,12 @@ litellm_settings:
enable_azure_ad_token_refresh: true # 👈 KEY CHANGE
```
3. Start proxy
```bash
litellm --config /path/to/config.yaml
```
</TabItem>
</Tabs>
@ -899,6 +948,65 @@ Expected Response:
{"data":[{"id":"batch_R3V...}
```
## O-Series Models
Azure OpenAI O-Series models are supported on LiteLLM.
LiteLLM routes any deployment name with `o1` or `o3` in the model name, to the O-Series [transformation](https://github.com/BerriAI/litellm/blob/91ed05df2962b8eee8492374b048d27cc144d08c/litellm/llms/azure/chat/o1_transformation.py#L4) logic.
To set this explicitly, set `model` to `azure/o_series/<your-deployment-name>`.
**Automatic Routing**
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
litellm.completion(model="azure/my-o3-deployment", messages=[{"role": "user", "content": "Hello, world!"}]) # 👈 Note: 'o3' in the deployment name
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: o3-mini
litellm_params:
model: azure/o3-model
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
```
</TabItem>
</Tabs>
**Explicit Routing**
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
litellm.completion(model="azure/o_series/my-random-deployment-name", messages=[{"role": "user", "content": "Hello, world!"}]) # 👈 Note: 'o_series/' in the deployment name
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: o3-mini
litellm_params:
model: azure/o_series/my-random-deployment-name
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
```
</TabItem>
</Tabs>
## Advanced
### Azure API Load-Balancing

195
docs/my-website/docs/providers/bedrock.md
View file

@ -2,7 +2,16 @@ import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# AWS Bedrock
ALL Bedrock models (Anthropic, Meta, Mistral, Amazon, etc.) are Supported
ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Supported
| Property | Details |
|-------|-------|
| Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#bedrock-imported-models-deepseek) |
| Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` |
| Pass-through Endpoint | [Supported](../pass_through/bedrock.md) |
LiteLLM requires `boto3` to be installed on your system for Bedrock requests
```shell
@ -792,6 +801,16 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
LiteLLM supports Document Understanding for Bedrock models - [AWS Bedrock Docs](https://docs.aws.amazon.com/nova/latest/userguide/modalities-document.html).
:::info
LiteLLM supports ALL Bedrock document types -
E.g.: "pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"
You can also pass these as either `image_url` or `base64`
:::
### url
<Tabs>
@ -1070,11 +1089,25 @@ response = completion(
)
```
### STS based Auth
### STS (Role-based Auth)
- Set `aws_role_name` and `aws_session_name`
| LiteLLM Parameter | Boto3 Parameter | Description | Boto3 Documentation |
|------------------|-----------------|-------------|-------------------|
| `aws_access_key_id` | `aws_access_key_id` | AWS access key associated with an IAM user or role | [Credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) |
| `aws_secret_access_key` | `aws_secret_access_key` | AWS secret key associated with the access key | [Credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) |
| `aws_role_name` | `RoleArn` | The Amazon Resource Name (ARN) of the role to assume | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
| `aws_session_name` | `RoleSessionName` | An identifier for the assumed role session | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
- Set `aws_role_name` and `aws_session_name` in completion() / embedding() function
Make the bedrock completion call
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
@ -1105,6 +1138,25 @@ response = completion(
aws_session_name="my-test-session",
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: bedrock/*
litellm_params:
model: bedrock/*
aws_role_name: arn:aws:iam::888602223428:role/iam_local_role # AWS RoleArn
aws_session_name: "bedrock-session" # AWS RoleSessionName
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # [OPTIONAL - not required if using role]
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # [OPTIONAL - not required if using role]
```
</TabItem>
</Tabs>
### Passing an external BedrockRuntime.Client as a parameter - Completion()
@ -1158,6 +1210,139 @@ response = completion(
aws_bedrock_client=bedrock,
)
```
## Calling via Internal Proxy
Use the `bedrock/converse_like/model` endpoint to call bedrock converse model via your internal proxy.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="bedrock/converse_like/some-model",
messages=[{"role": "user", "content": "What's AWS?"}],
api_key="sk-1234",
api_base="https://some-api-url/models",
extra_headers={"test": "hello world"},
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
1. Setup config.yaml
```yaml
model_list:
- model_name: anthropic-claude
litellm_params:
model: bedrock/converse_like/some-model
api_base: https://some-api-url/models
```
2. Start proxy server
```bash
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "anthropic-claude",
"messages": [
{
"role": "system",
"content": "You are a helpful math tutor. Guide the user through the solution step by step."
},
{ "content": "Hello, how are you?", "role": "user" }
]
}'
```
</TabItem>
</Tabs>
**Expected Output URL**
```bash
https://some-api-url/models
```
## Bedrock Imported Models (Deepseek)
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/llama/{model_arn}` |
| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) |
Use this route to call Bedrock Imported Models that follow the `llama` Invoke Request / Response spec
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
response = completion(
model="bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/llama/{your-model-arn}
messages=[{"role": "user", "content": "Tell me a joke"}],
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: DeepSeek-R1-Distill-Llama-70B
litellm_params:
model: bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
## Provisioned throughput models
@ -1372,4 +1557,6 @@ curl http://0.0.0.0:4000/rerank \
```
</TabItem>
</Tabs>
</Tabs>

72
docs/my-website/docs/providers/deepseek.md
View file

@ -1,3 +1,6 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Deepseek
https://deepseek.com/
@ -52,3 +55,72 @@ We support ALL Deepseek models, just set `deepseek/` as a prefix when sending co
| deepseek-coder | `completion(model="deepseek/deepseek-coder", messages)` |
## Reasoning Models
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| deepseek-reasoner | `completion(model="deepseek/deepseek-reasoner", messages)` |
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ['DEEPSEEK_API_KEY'] = ""
resp = completion(
model="deepseek/deepseek-reasoner",
messages=[{"role": "user", "content": "Tell me a joke."}],
)
print(
resp.choices[0].message.reasoning_content
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: deepseek-reasoner
litellm_params:
model: deepseek/deepseek-reasoner
api_key: os.environ/DEEPSEEK_API_KEY
```
2. Run proxy
```bash
python litellm/proxy/main.py
```
3. Test it!
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "deepseek-reasoner",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Hi, how are you ?"
}
]
}
]
}'
```
</TabItem>
</Tabs>

35
docs/my-website/docs/providers/friendliai.md
View file

@ -1,23 +1,32 @@
# FriendliAI
https://suite.friendli.ai/
:::info
**We support ALL FriendliAI models, just set `friendliai/` as a prefix when sending completion requests**
:::
| Property | Details |
| -------------------------- | ----------------------------------------------------------------------------------------------- |
| Description | The fastest and most efficient inference engine to build production-ready, compound AI systems. |
| Provider Route on LiteLLM | `friendliai/` |
| Provider Doc | [FriendliAI ↗](https://friendli.ai/docs/sdk/integrations/litellm) |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions` |
## API Key
```python
# env variable
os.environ['FRIENDLI_TOKEN']
os.environ['FRIENDLI_API_BASE'] # Optional. Set this when using dedicated endpoint.
```
## Sample Usage
```python
from litellm import completion
import os
os.environ['FRIENDLI_TOKEN'] = ""
response = completion(
model="friendliai/mixtral-8x7b-instruct-v0-1",
model="friendliai/meta-llama-3.1-8b-instruct",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@ -26,13 +35,14 @@ print(response)
```
## Sample Usage - Streaming
```python
from litellm import completion
import os
os.environ['FRIENDLI_TOKEN'] = ""
response = completion(
model="friendliai/mixtral-8x7b-instruct-v0-1",
model="friendliai/meta-llama-3.1-8b-instruct",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@ -43,18 +53,11 @@ for chunk in response:
print(chunk)
```
## Supported Models
### Serverless Endpoints
We support ALL FriendliAI AI models, just set `friendliai/` as a prefix when sending completion requests
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| mixtral-8x7b-instruct | `completion(model="friendliai/mixtral-8x7b-instruct-v0-1", messages)` |
| meta-llama-3-8b-instruct | `completion(model="friendliai/meta-llama-3-8b-instruct", messages)` |
| meta-llama-3-70b-instruct | `completion(model="friendliai/meta-llama-3-70b-instruct", messages)` |
### Dedicated Endpoints
```
model="friendliai/$ENDPOINT_ID:$ADAPTER_ROUTE"
```
| Model Name | Function Call |
| --------------------------- | ---------------------------------------------------------------------- |
| meta-llama-3.1-8b-instruct | `completion(model="friendliai/meta-llama-3.1-8b-instruct", messages)` |
| meta-llama-3.1-70b-instruct | `completion(model="friendliai/meta-llama-3.1-70b-instruct", messages)` |

24
docs/my-website/docs/providers/lm_studio.md
View file

@ -11,6 +11,14 @@ https://lmstudio.ai/docs/basics/server
:::
| Property | Details |
|-------|-------|
| Description | Discover, download, and run local LLMs. |
| Provider Route on LiteLLM | `lm_studio/` |
| Provider Doc | [LM Studio ↗](https://lmstudio.ai/docs/api/openai-api) |
| Supported OpenAI Endpoints | `/chat/completions`, `/embeddings`, `/completions` |
## API Key
```python
# env variable
@ -42,7 +50,7 @@ print(response)
from litellm import completion
import os
os.environ['XAI_API_KEY'] = ""
os.environ['LM_STUDIO_API_KEY'] = ""
response = completion(
model="lm_studio/llama-3-8b-instruct",
messages=[
@ -131,3 +139,17 @@ Here's how to call a XAI model with the LiteLLM Proxy Server
## Supported Parameters
See [Supported Parameters](../completion/input.md#translated-openai-params) for supported parameters.
## Embedding
```python
from litellm import embedding
import os
os.environ['LM_STUDIO_API_BASE'] = "http://localhost:8000"
response = embedding(
model="lm_studio/jina-embeddings-v3",
input=["Hello world"],
)
print(response)
```

79
docs/my-website/docs/providers/ollama.md
View file

@ -147,6 +147,7 @@ model_list:
- model_name: "llama3.1"
litellm_params:
model: "ollama_chat/llama3.1"
keep_alive: "8m" # Optional: Overrides default keep_alive, use -1 for Forever
model_info:
supports_function_calling: true
```
@ -237,6 +238,76 @@ Ollama supported models: https://github.com/ollama/ollama
| Nous-Hermes 13B | `completion(model='ollama/nous-hermes:13b', messages, api_base="http://localhost:11434", stream=True)` |
| Wizard Vicuna Uncensored | `completion(model='ollama/wizard-vicuna', messages, api_base="http://localhost:11434", stream=True)` |
### JSON Schema support
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="ollama_chat/deepseek-r1",
messages=[{ "content": "respond in 20 words. who are you?","role": "user"}],
response_format={"type": "json_schema", "json_schema": {"schema": {"type": "object", "properties": {"name": {"type": "string"}}}}},
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: "deepseek-r1"
litellm_params:
model: "ollama_chat/deepseek-r1"
api_base: "http://localhost:11434"
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING ON http://0.0.0.0:4000
```
3. Test it!
```python
from pydantic import BaseModel
from openai import OpenAI
client = OpenAI(
api_key="anything", # 👈 PROXY KEY (can be anything, if master_key not set)
base_url="http://0.0.0.0:4000" # 👈 PROXY BASE URL
)
class Step(BaseModel):
explanation: str
output: str
class MathReasoning(BaseModel):
steps: list[Step]
final_answer: str
completion = client.beta.chat.completions.parse(
model="deepseek-r1",
messages=[
{"role": "system", "content": "You are a helpful math tutor. Guide the user through the solution step by step."},
{"role": "user", "content": "how can I solve 8x + 7 = -23"}
],
response_format=MathReasoning,
)
math_reasoning = completion.choices[0].message.parsed
```
</TabItem>
</Tabs>
## Ollama Vision Models
| Model Name | Function Call |
|------------------|--------------------------------------|
@ -355,8 +426,6 @@ for chunk in response:
}
```
## Support / talk with founders
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
## Calling Docker Container (host.docker.internal)
[Follow these instructions](https://github.com/BerriAI/litellm/issues/1517#issuecomment-1922022209/)

27
docs/my-website/docs/providers/topaz.md Normal file
View file

@ -0,0 +1,27 @@
# Topaz
| Property | Details |
|-------|-------|
| Description | Professional-grade photo and video editing powered by AI. |
| Provider Route on LiteLLM | `topaz/` |
| Provider Doc | [Topaz ↗](https://www.topazlabs.com/enhance-api) |
| API Endpoint for Provider | https://api.topazlabs.com |
| Supported OpenAI Endpoints | `/image/variations` |
## Quick Start
```python
from litellm import image_variation
import os
os.environ["TOPAZ_API_KEY"] = ""
response = image_variation(
model="topaz/Standard V2", image=image_url
)
```
## Supported OpenAI Params
- `response_format`
- `size` (widthxheight)

146
docs/my-website/docs/providers/vertex.md
View file

@ -4,6 +4,7 @@ import TabItem from '@theme/TabItem';
# VertexAI [Anthropic, Gemini, Model Garden]
## Overview
| Property | Details |
|-------|-------|
@ -11,6 +12,8 @@ import TabItem from '@theme/TabItem';
| Provider Route on LiteLLM | `vertex_ai/` |
| Link to Provider Doc | [Vertex AI ↗](https://cloud.google.com/vertex-ai) |
| Base URL | [https://{vertex_location}-aiplatform.googleapis.com/](https://{vertex_location}-aiplatform.googleapis.com/) |
| Supported Operations | [`/chat/completions`](#sample-usage), `/completions`, [`/embeddings`](#embedding-models), [`/audio/speech`](#text-to-speech-apis), [`/fine_tuning`](#fine-tuning-apis), [`/batches`](#batch-apis), [`/files`](#batch-apis), [`/images`](#image-generation-models) |
<br />
<br />
@ -2477,7 +2480,7 @@ create_batch_response = oai_client.batches.create(
```json
{
"id": "projects/633608382793/locations/us-central1/batchPredictionJobs/986266568679751680",
"id": "3814889423749775360",
"completion_window": "24hrs",
"created_at": 1733392026,
"endpoint": "",
@ -2500,6 +2503,147 @@ create_batch_response = oai_client.batches.create(
}
```
#### 4. Retrieve a batch
```python
retrieved_batch = oai_client.batches.retrieve(
batch_id=create_batch_response.id,
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use `vertex_ai` for this batch request
)
```
**Expected Response**
```json
{
"id": "3814889423749775360",
"completion_window": "24hrs",
"created_at": 1736500100,
"endpoint": "",
"input_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/7b2e47f5-3dd4-436d-920f-f9155bbdc952",
"object": "batch",
"status": "completed",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001",
"request_counts": null
}
```
## **Fine Tuning APIs**
| Property | Details |
|----------|---------|
| Description | Create Fine Tuning Jobs in Vertex AI (`/tuningJobs`) using OpenAI Python SDK |
| Vertex Fine Tuning Documentation | [Vertex Fine Tuning](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/tuning#create-tuning) |
### Usage
#### 1. Add `finetune_settings` to your config.yaml
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: openai/fake
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
# 👇 Key change: For /fine_tuning/jobs endpoints
finetune_settings:
- custom_llm_provider: "vertex_ai"
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: "/Users/ishaanjaffer/Downloads/adroit-crow-413218-a956eef1a2a8.json"
```
#### 2. Create a Fine Tuning Job
<Tabs>
<TabItem value="openai" label="OpenAI Python SDK">
```python
ft_job = await client.fine_tuning.jobs.create(
model="gemini-1.0-pro-002", # Vertex model you want to fine-tune
training_file="gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl", # file_id from create file response
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm proxy which provider to use
)
```
</TabItem>
<TabItem value="curl" label="curl">
```shell
curl http://localhost:4000/v1/fine_tuning/jobs \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"custom_llm_provider": "vertex_ai",
"model": "gemini-1.0-pro-002",
"training_file": "gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl"
}'
```
</TabItem>
</Tabs>
**Advanced use case - Passing `adapter_size` to the Vertex AI API**
Set hyper_parameters, such as `n_epochs`, `learning_rate_multiplier` and `adapter_size`. [See Vertex Advanced Hyperparameters](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/tuning#advanced_use_case)
<Tabs>
<TabItem value="openai" label="OpenAI Python SDK">
```python
ft_job = client.fine_tuning.jobs.create(
model="gemini-1.0-pro-002", # Vertex model you want to fine-tune
training_file="gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl", # file_id from create file response
hyperparameters={
"n_epochs": 3, # epoch_count on Vertex
"learning_rate_multiplier": 0.1, # learning_rate_multiplier on Vertex
"adapter_size": "ADAPTER_SIZE_ONE" # type: ignore, vertex specific hyperparameter
},
extra_body={
"custom_llm_provider": "vertex_ai",
},
)
```
</TabItem>
<TabItem value="curl" label="curl">
```shell
curl http://localhost:4000/v1/fine_tuning/jobs \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"custom_llm_provider": "vertex_ai",
"model": "gemini-1.0-pro-002",
"training_file": "gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl",
"hyperparameters": {
"n_epochs": 3,
"learning_rate_multiplier": 0.1,
"adapter_size": "ADAPTER_SIZE_ONE"
}
}'
```
</TabItem>
</Tabs>
## Extra

1
docs/my-website/docs/providers/watsonx.md
View file

@ -14,6 +14,7 @@ os.environ["WATSONX_TOKEN"] = "" # IAM auth token
# optional - can also be passed as params to completion() or embedding()
os.environ["WATSONX_PROJECT_ID"] = "" # Project ID of your WatsonX instance
os.environ["WATSONX_DEPLOYMENT_SPACE_ID"] = "" # ID of your deployment space to use deployed models
os.environ["WATSONX_ZENAPIKEY"] = "" # Zen API key (use for long-term api token)
```
See [here](https://cloud.ibm.com/apidocs/watsonx-ai#api-authentication) for more information on how to get an access token to authenticate to watsonx.ai.

4
docs/my-website/docs/proxy/admin_ui_sso.md
View file

@ -1,3 +1,7 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# ✨ SSO for Admin UI
:::info

103
docs/my-website/docs/proxy/alerting.md
View file

@ -6,17 +6,13 @@ import TabItem from '@theme/TabItem';
Get alerts for:
- Hanging LLM api calls
- Slow LLM api calls
- Failed LLM api calls
- Budget Tracking per key/user
- Spend Reports - Weekly & Monthly spend per Team, Tag
- Failed db read/writes
- Model outage alerting
- Daily Reports:
- **LLM** Top 5 slowest deployments
- **LLM** Top 5 deployments with most failed requests
- **Spend** Weekly & Monthly spend per Team, Tag
| Category | Alert Type |
|----------|------------|
| **LLM Performance** | Hanging API calls, Slow API calls, Failed API calls, Model outage alerting |
| **Budget & Spend** | Budget tracking per key/user, Soft budget alerts, Weekly & Monthly spend reports per Team/Tag |
| **System Health** | Failed database read/writes |
| **Daily Reports** | Top 5 slowest LLM deployments, Top 5 LLM deployments with most failed requests, Weekly & Monthly spend per Team/Tag |
Works across:
@ -47,7 +43,20 @@ export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/<>/<>/<>"
general_settings:
alerting: ["slack"]
alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+
spend_report_frequency: "1d" # [Optional] set as 1d, 2d, 30d .... Specifiy how often you want a Spend Report to be sent
spend_report_frequency: "1d" # [Optional] set as 1d, 2d, 30d .... Specify how often you want a Spend Report to be sent
# [OPTIONAL ALERTING ARGS]
alerting_args:
daily_report_frequency: 43200 # 12 hours in seconds
report_check_interval: 3600 # 1 hour in seconds
budget_alert_ttl: 86400 # 24 hours in seconds
outage_alert_ttl: 60 # 1 minute in seconds
region_outage_alert_ttl: 60 # 1 minute in seconds
minor_outage_alert_threshold: 5
major_outage_alert_threshold: 10
max_outage_alert_list_size: 1000
log_to_console: false
```
Start proxy
@ -80,6 +89,51 @@ litellm_settings:
redact_messages_in_exceptions: True
```
### Soft Budget Alerts for Virtual Keys
Use this to send an alert when a key/team is close to it's budget running out
Step 1. Create a virtual key with a soft budget
Set the `soft_budget` to 0.001
```shell
curl -X 'POST' \
'http://localhost:4000/key/generate' \
-H 'accept: application/json' \
-H 'x-goog-api-key: sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"key_alias": "prod-app1",
"team_id": "113c1a22-e347-4506-bfb2-b320230ea414",
"soft_budget": 0.001
}'
```
Step 2. Send a request to the proxy with the virtual key
```shell
curl http://0.0.0.0:4000/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-Nb5eCf427iewOlbxXIH4Ow" \
-d '{
"model": "openai/gpt-4",
"messages": [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
]
}'
```
Step 3. Check slack for Expected Alert
<Image img={require('../../img/soft_budget_alert.png')}/>
### Add Metadata to alerts
@ -110,7 +164,7 @@ response = client.chat.completions.create(
<Image img={require('../../img/alerting_metadata.png')}/>
### Opting into specific alert types
### Select specific alert types
Set `alert_types` if you want to Opt into only specific alert types. When alert_types is not set, all Default Alert Types are enabled.
@ -132,7 +186,7 @@ general_settings:
]
```
### Set specific slack channels per alert type
### Map slack channels to alert type
Use this if you want to set specific channels per alert type
@ -230,7 +284,7 @@ curl -i http://localhost:4000/v1/chat/completions \
```
### Using MS Teams Webhooks
### MS Teams Webhooks
MS Teams provides a slack compatible webhook url that you can use for alerting
@ -272,7 +326,7 @@ curl --location 'http://0.0.0.0:4000/health/services?service=slack' \
<Image img={require('../../img/ms_teams_alerting.png')}/>
### Using Discord Webhooks
### Discord Webhooks
Discord provides a slack compatible webhook url that you can use for alerting
@ -456,4 +510,19 @@ Management Endpoint Alerts - Virtual Key, Team, Internal User
| `team_deleted` | Alerts when a team is deleted | ❌ |
| `new_internal_user_created` | Notifications for new internal user accounts | ❌ |
| `internal_user_updated` | Alerts when an internal user's details are changed | ❌ |
| `internal_user_deleted` | Notifications when an internal user account is removed | ❌ |
| `internal_user_deleted` | Notifications when an internal user account is removed | ❌ |
## `alerting_args` Specification
| Parameter | Default | Description |
|-----------|---------|-------------|
| `daily_report_frequency` | 43200 (12 hours) | Frequency of receiving deployment latency/failure reports in seconds |
| `report_check_interval` | 3600 (1 hour) | How often to check if a report should be sent (background process) in seconds |
| `budget_alert_ttl` | 86400 (24 hours) | Cache TTL for budget alerts to prevent spam when budget is crossed |
| `outage_alert_ttl` | 60 (1 minute) | Time window for collecting model outage errors in seconds |
| `region_outage_alert_ttl` | 60 (1 minute) | Time window for collecting region-based outage errors in seconds |
| `minor_outage_alert_threshold` | 5 | Number of errors that trigger a minor outage alert (400 errors not counted) |
| `major_outage_alert_threshold` | 10 | Number of errors that trigger a major outage alert (400 errors not counted) |
| `max_outage_alert_list_size` | 1000 | Maximum number of errors to store in cache per model/region |
| `log_to_console` | false | If true, prints alerting payload to console as a `.warning` log. |

2
docs/my-website/docs/proxy/architecture.md
View file

@ -30,7 +30,7 @@ import TabItem from '@theme/TabItem';
6. [**litellm.completion() / litellm.embedding()**:](../index#litellm-python-sdk) The litellm Python SDK is used to call the LLM in the OpenAI API format (Translation and parameter mapping)
7. **Post-Request Processing**: After the response is sent back to the client, the following **asynchronous** tasks are performed:
- [Logging to LangFuse (logging destination is configurable)](./logging)
- [Logging to Lunary, MLflow, LangFuse or other logging destinations](./logging)
- The [MaxParallelRequestsHandler](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/parallel_request_limiter.py) updates the rpm/tpm usage for the
- Global Server Rate Limit
- Virtual Key Rate Limit

154
docs/my-website/docs/proxy/bucket.md
View file

@ -1,154 +0,0 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Logging GCS, s3 Buckets
LiteLLM Supports Logging to the following Cloud Buckets
- (Enterprise) ✨ [Google Cloud Storage Buckets](#logging-proxy-inputoutput-to-google-cloud-storage-buckets)
- (Free OSS) [Amazon s3 Buckets](#logging-proxy-inputoutput---s3-buckets)
## Google Cloud Storage Buckets
Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage?hl=en)
:::info
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
:::
| Property | Details |
|----------|---------|
| Description | Log LLM Input/Output to cloud storage buckets |
| Load Test Benchmarks | [Benchmarks](https://docs.litellm.ai/docs/benchmarks) |
| Google Docs on Cloud Storage | [Google Cloud Storage](https://cloud.google.com/storage?hl=en) |
### Usage
1. Add `gcs_bucket` to LiteLLM Config.yaml
```yaml
model_list:
- litellm_params:
api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/
api_key: my-fake-key
model: openai/my-fake-model
model_name: fake-openai-endpoint
litellm_settings:
callbacks: ["gcs_bucket"] # 👈 KEY CHANGE # 👈 KEY CHANGE
```
2. Set required env variables
```shell
GCS_BUCKET_NAME="<your-gcs-bucket-name>"
GCS_PATH_SERVICE_ACCOUNT="/Users/ishaanjaffer/Downloads/adroit-crow-413218-a956eef1a2a8.json" # Add path to service account.json
```
3. Start Proxy
```
litellm --config /path/to/config.yaml
```
4. Test it!
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data ' {
"model": "fake-openai-endpoint",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}
'
```
### Expected Logs on GCS Buckets
<Image img={require('../../img/gcs_bucket.png')} />
### Fields Logged on GCS Buckets
[**The standard logging object is logged on GCS Bucket**](../proxy/logging)
### Getting `service_account.json` from Google Cloud Console
1. Go to [Google Cloud Console](https://console.cloud.google.com/)
2. Search for IAM & Admin
3. Click on Service Accounts
4. Select a Service Account
5. Click on 'Keys' -> Add Key -> Create New Key -> JSON
6. Save the JSON file and add the path to `GCS_PATH_SERVICE_ACCOUNT`
## s3 Buckets
We will use the `--config` to set
- `litellm.success_callback = ["s3"]`
This will log all successfull LLM calls to s3 Bucket
**Step 1** Set AWS Credentials in .env
```shell
AWS_ACCESS_KEY_ID = ""
AWS_SECRET_ACCESS_KEY = ""
AWS_REGION_NAME = ""
```
**Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback`
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
litellm_settings:
success_callback: ["s3"]
s3_callback_params:
s3_bucket_name: logs-bucket-litellm # AWS Bucket Name for S3
s3_region_name: us-west-2 # AWS Region Name for S3
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets
```
**Step 3**: Start the proxy, make a test request
Start proxy
```shell
litellm --config config.yaml --debug
```
Test Request
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data ' {
"model": "Azure OpenAI GPT-4 East",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
Your logs should be available on the specified s3 Bucket

3
docs/my-website/docs/proxy/call_hooks.md
View file

@ -139,9 +139,6 @@ class MyCustomHandler(CustomLogger): # https://docs.litellm.ai/docs/observabilit
#### ASYNC ####
async def async_log_stream_event(self, kwargs, response_obj, start_time, end_time):
pass
async def async_log_pre_api_call(self, model, messages, kwargs):
pass

18
docs/my-website/docs/proxy/config_settings.md
View file

@ -139,6 +139,7 @@ general_settings:
| disable_end_user_cost_tracking_prometheus_only | boolean | If true, turns off end user cost tracking on prometheus metrics only. |
| key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) |
| disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. |
| disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
### general_settings - Reference
@ -177,6 +178,7 @@ general_settings:
| service_account_settings | List[Dict[str, Any]] | Set `service_account_settings` if you want to create settings that only apply to service account keys (Doc on service accounts)[./service_accounts.md] |
| image_generation_model | str | The default model to use for image generation - ignores model set in request |
| store_model_in_db | boolean | If true, allows `/model/new` endpoint to store model information in db. Endpoint disabled by default. [Doc on `/model/new` endpoint](./model_management.md#create-a-new-model) |
| store_prompts_in_spend_logs | boolean | If true, allows prompts and responses to be stored in the spend logs table. |
| max_request_size_mb | int | The maximum size for requests in MB. Requests above this size will be rejected. |
| max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. |
| proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** |
@ -222,7 +224,7 @@ router_settings:
redis_host: <your-redis-host> # string
redis_password: <your-redis-password> # string
redis_port: <your-redis-port> # string
enable_pre_call_check: true # bool - Before call is made check if a call is within model context window
enable_pre_call_checks: true # bool - Before call is made check if a call is within model context window
allowed_fails: 3 # cooldown model if it fails > 1 call in a minute.
cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
disable_cooldowns: True # bool - Disable cooldowns for all models
@ -266,7 +268,8 @@ router_settings:
| polling_interval | (Optional[float]) | frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms. |
| max_fallbacks | Optional[int] | The maximum number of fallbacks to try before exiting the call. Defaults to 5. |
| default_litellm_params | Optional[dict] | The default litellm parameters to add to all requests (e.g. `temperature`, `max_tokens`). |
| timeout | Optional[float] | The default timeout for a request. |
| timeout | Optional[float] | The default timeout for a request. Default is 10 minutes. |
| stream_timeout | Optional[float] | The default timeout for a streaming request. If not set, the 'timeout' value is used. |
| debug_level | Literal["DEBUG", "INFO"] | The debug level for the logging library in the router. Defaults to "INFO". |
| client_ttl | int | Time-to-live for cached clients in seconds. Defaults to 3600. |
| cache_kwargs | dict | Additional keyword arguments for the cache initialization. |
@ -306,6 +309,7 @@ router_settings:
| ARGILLA_DATASET_NAME | Dataset name for Argilla logging
| ARGILLA_BASE_URL | Base URL for Argilla service
| ATHINA_API_KEY | API key for Athina service
| ATHINA_BASE_URL | Base URL for Athina service (defaults to `https://log.athina.ai`)
| AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key)
| AWS_ACCESS_KEY_ID | Access Key ID for AWS services
| AWS_PROFILE_NAME | AWS CLI profile name to be used
@ -364,6 +368,8 @@ router_settings:
| GCS_PATH_SERVICE_ACCOUNT | Path to the Google Cloud service account JSON file
| GCS_FLUSH_INTERVAL | Flush interval for GCS logging (in seconds). Specify how often you want a log to be sent to GCS. **Default is 20 seconds**
| GCS_BATCH_SIZE | Batch size for GCS logging. Specify after how many logs you want to flush to GCS. If `BATCH_SIZE` is set to 10, logs are flushed every 10 logs. **Default is 2048**
| GCS_PUBSUB_TOPIC_ID | PubSub Topic ID to send LiteLLM SpendLogs to.
| GCS_PUBSUB_PROJECT_ID | PubSub Project ID to send LiteLLM SpendLogs to.
| GENERIC_AUTHORIZATION_ENDPOINT | Authorization endpoint for generic OAuth providers
| GENERIC_CLIENT_ID | Client ID for generic OAuth providers
| GENERIC_CLIENT_SECRET | Client secret for generic OAuth providers
@ -390,6 +396,12 @@ router_settings:
| GOOGLE_CLIENT_SECRET | Client secret for Google OAuth
| GOOGLE_KMS_RESOURCE_NAME | Name of the resource in Google KMS
| HF_API_BASE | Base URL for Hugging Face API
| HCP_VAULT_ADDR | Address for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_CLIENT_CERT | Path to client certificate for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_CLIENT_KEY | Path to client key for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_NAMESPACE | Namespace for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_TOKEN | Token for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_CERT_ROLE | Role for [Hashicorp Vault Secret Manager Auth](../secret.md#hashicorp-vault)
| HELICONE_API_KEY | API key for Helicone service
| HOSTNAME | Hostname for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog)
| HUGGINGFACE_API_BASE | Base URL for Hugging Face API
@ -430,6 +442,7 @@ router_settings:
| LITELLM_SALT_KEY | Salt key for encryption in LiteLLM
| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
| LITELLM_TOKEN | Access token for LiteLLM integration
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LOGFIRE_TOKEN | Token for Logfire logging service
| MICROSOFT_CLIENT_ID | Client ID for Microsoft services
| MICROSOFT_CLIENT_SECRET | Client secret for Microsoft services
@ -452,6 +465,7 @@ router_settings:
| OTEL_HEADERS | Headers for OpenTelemetry requests
| OTEL_SERVICE_NAME | Service name identifier for OpenTelemetry
| OTEL_TRACER_NAME | Tracer name for OpenTelemetry tracing
| PAGERDUTY_API_KEY | API key for PagerDuty Alerting
| POD_NAME | Pod name for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog) as `POD_NAME`
| PREDIBASE_API_BASE | Base URL for Predibase API
| PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service

26
docs/my-website/docs/proxy/configs.md
View file

@ -516,6 +516,32 @@ model_list:
$ litellm --config /path/to/config.yaml
```
### Set custom tokenizer
If you're using the [`/utils/token_counter` endpoint](https://litellm-api.up.railway.app/#/llm%20utils/token_counter_utils_token_counter_post), and want to set a custom huggingface tokenizer for a model, you can do so in the `config.yaml`
```yaml
model_list:
- model_name: openai-deepseek
litellm_params:
model: deepseek/deepseek-chat
api_key: os.environ/OPENAI_API_KEY
model_info:
access_groups: ["restricted-models"]
custom_tokenizer:
identifier: deepseek-ai/DeepSeek-V3-Base
revision: main
auth_token: os.environ/HUGGINGFACE_API_KEY
```
**Spec**
```
custom_tokenizer:
identifier: str # huggingface model identifier
revision: str # huggingface model revision (usually 'main')
auth_token: Optional[str] # huggingface auth token
```
## General Settings `general_settings` (DB Connection, etc)
### Configure DB Pool Limits + Connection Timeouts

48
docs/my-website/docs/proxy/custom_auth.md Normal file
View file

@ -0,0 +1,48 @@
# Custom Auth
You can now override the default api key auth.
Here's how:
#### 1. Create a custom auth file.
Make sure the response type follows the `UserAPIKeyAuth` pydantic object. This is used by for logging usage specific to that user key.
```python
from litellm.proxy._types import UserAPIKeyAuth
async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
try:
modified_master_key = "sk-my-master-key"
if api_key == modified_master_key:
return UserAPIKeyAuth(api_key=api_key)
raise Exception
except:
raise Exception
```
#### 2. Pass the filepath (relative to the config.yaml)
Pass the filepath to the config.yaml
e.g. if they're both in the same dir - `./config.yaml` and `./custom_auth.py`, this is what it looks like:
```yaml
model_list:
- model_name: "openai-model"
litellm_params:
model: "gpt-3.5-turbo"
litellm_settings:
drop_params: True
set_verbose: True
general_settings:
custom_auth: custom_auth.user_api_key_auth
```
[**Implementation Code**](https://github.com/BerriAI/litellm/blob/caf2a6b279ddbe89ebd1d8f4499f65715d684851/litellm/proxy/utils.py#L122)
#### 3. Start the proxy
```shell
$ litellm --config /path/to/config.yaml
```

142
docs/my-website/docs/proxy/deploy.md
View file

@ -32,11 +32,10 @@ source .env
docker-compose up
```
<Tabs>
<TabItem value="basic" label="Basic (No DB)">
### Docker Run
### Step 1. CREATE config.yaml
#### Step 1. CREATE config.yaml
Example `litellm_config.yaml`
@ -52,7 +51,7 @@ model_list:
### Step 2. RUN Docker Image
#### Step 2. RUN Docker Image
```shell
docker run \
@ -66,7 +65,7 @@ docker run \
Get Latest Image 👉 [here](https://github.com/berriai/litellm/pkgs/container/litellm)
### Step 3. TEST Request
#### Step 3. TEST Request
Pass `model=azure-gpt-3.5` this was set on step 1
@ -84,13 +83,7 @@ Get Latest Image 👉 [here](https://github.com/berriai/litellm/pkgs/container/l
}'
```
</TabItem>
<TabItem value="cli" label="With CLI Args">
#### Run with LiteLLM CLI args
### Docker Run - CLI Args
See all supported CLI args [here](https://docs.litellm.ai/docs/proxy/cli):
@ -104,15 +97,8 @@ Here's how you can run the docker image and start litellm on port 8002 with `num
docker run ghcr.io/berriai/litellm:main-latest --port 8002 --num_workers 8
```
</TabItem>
<TabItem value="terraform" label="Terraform">
s/o [Nicholas Cecere](https://www.linkedin.com/in/nicholas-cecere-24243549/) for his LiteLLM User Management Terraform
👉 [Go here for Terraform](https://github.com/ncecere/terraform-litellm-user-mgmt)
</TabItem>
<TabItem value="base-image" label="use litellm as a base image">
### Use litellm as a base image
```shell
# Use the provided base image
@ -137,9 +123,75 @@ EXPOSE 4000/tcp
CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]
```
</TabItem>
### Build from litellm `pip` package
<TabItem value="kubernetes" label="Kubernetes">
Follow these instructons to build a docker container from the litellm pip package. If your company has a strict requirement around security / building images you can follow these steps.
Dockerfile
```shell
FROM cgr.dev/chainguard/python:latest-dev
USER root
WORKDIR /app
ENV HOME=/home/litellm
ENV PATH="${HOME}/venv/bin:$PATH"
# Install runtime dependencies
RUN apk update && \
apk add --no-cache gcc python3-dev openssl openssl-dev
RUN python -m venv ${HOME}/venv
RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip
COPY requirements.txt .
RUN --mount=type=cache,target=${HOME}/.cache/pip \
${HOME}/venv/bin/pip install -r requirements.txt
EXPOSE 4000/tcp
ENTRYPOINT ["litellm"]
CMD ["--port", "4000"]
```
Example `requirements.txt`
```shell
litellm[proxy]==1.57.3 # Specify the litellm version you want to use
prometheus_client
langfuse
prisma
```
Build the docker image
```shell
docker build \
-f Dockerfile.build_from_pip \
-t litellm-proxy-with-pip-5 .
```
Run the docker image
```shell
docker run \
-v $(pwd)/litellm_config.yaml:/app/config.yaml \
-e OPENAI_API_KEY="sk-1222" \
-e DATABASE_URL="postgresql://xxxxxxxxx \
-p 4000:4000 \
litellm-proxy-with-pip-5 \
--config /app/config.yaml --detailed_debug
```
### Terraform
s/o [Nicholas Cecere](https://www.linkedin.com/in/nicholas-cecere-24243549/) for his LiteLLM User Management Terraform
👉 [Go here for Terraform](https://github.com/ncecere/terraform-litellm-user-mgmt)
### Kubernetes
Deploying a config file based litellm instance just requires a simple deployment that loads
the config.yaml file via a config map. Also it would be a good practice to use the env var
@ -204,11 +256,8 @@ spec:
To avoid issues with predictability, difficulties in rollback, and inconsistent environments, use versioning or SHA digests (for example, `litellm:main-v1.30.3` or `litellm@sha256:12345abcdef...`) instead of `litellm:main-latest`.
:::
</TabItem>
<TabItem value="helm-" label="Helm Chart">
### Helm Chart
:::info
@ -248,13 +297,9 @@ kubectl --namespace default port-forward $POD_NAME 8080:$CONTAINER_PORT
Your LiteLLM Proxy Server is now running on `http://127.0.0.1:4000`.
</TabItem>
</Tabs>
**That's it ! That's the quick start to deploy litellm**
## Use with Langchain, OpenAI SDK, LlamaIndex, Instructor, Curl
#### Make LLM API Requests
:::info
💡 Go here 👉 [to make your first LLM API Request](user_keys)
@ -263,7 +308,7 @@ LiteLLM is compatible with several SDKs - including OpenAI SDK, Anthropic SDK, M
:::
## Options to deploy LiteLLM
## Deployment Options
| Docs | When to Use |
| ------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- |
@ -272,8 +317,8 @@ LiteLLM is compatible with several SDKs - including OpenAI SDK, Anthropic SDK, M
| [LiteLLM container + Redis](#litellm-container--redis) | + load balance across multiple litellm containers |
| [LiteLLM Database container + PostgresDB + Redis](#litellm-database-container--postgresdb--redis) | + use Virtual Keys + Track Spend + load balance across multiple litellm containers |
## Deploy with Database
### Docker, Kubernetes, Helm Chart
### Deploy with Database
##### Docker, Kubernetes, Helm Chart
Requirements:
- Need a postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), etc) Set `DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname>` in your env
@ -491,7 +536,7 @@ Your LiteLLM Proxy Server is now running on `http://127.0.0.1:4000`.
</TabItem>
</Tabs>
## LiteLLM container + Redis
### Deploy with Redis
Use Redis when you need litellm to load balance across multiple litellm containers
The only change required is setting Redis on your `config.yaml`
@ -523,7 +568,7 @@ Start docker container with config
docker run ghcr.io/berriai/litellm:main-latest --config your_config.yaml
```
## LiteLLM Database container + PostgresDB + Redis
### Deploy with Database + Redis
The only change required is setting Redis on your `config.yaml`
LiteLLM Proxy supports sharing rpm/tpm shared across multiple litellm instances, pass `redis_host`, `redis_password` and `redis_port` to enable this. (LiteLLM will use Redis to track rpm/tpm usage )
@ -558,7 +603,7 @@ docker run --name litellm-proxy \
ghcr.io/berriai/litellm-database:main-latest --config your_config.yaml
```
## LiteLLM without Internet Connection
### (Non Root) - without Internet Connection
By default `prisma generate` downloads [prisma's engine binaries](https://www.prisma.io/docs/orm/reference/environment-variables-reference#custom-engine-file-locations). This might cause errors when running without internet connection.
@ -572,7 +617,7 @@ docker pull ghcr.io/berriai/litellm-non_root:main-stable
## Advanced Deployment Settings
### 1. Customization of the server root path (custom Proxy base url)
### 1. Custom server root path (Proxy base url)
💥 Use this when you want to serve LiteLLM on a custom base url path like `https://localhost:4000/api/v1`
@ -670,7 +715,7 @@ After running the proxy you can access it on `http://0.0.0.0:4000/api/v1/` (sinc
**That's it**, that's all you need to run the proxy on a custom root path
### 2. Setting SSL Certification
### 2. SSL Certification
Use this, If you need to set ssl certificates for your on prem litellm proxy
@ -684,7 +729,7 @@ docker run ghcr.io/berriai/litellm:main-latest \
Provide an ssl certificate when starting litellm proxy server
### 3. Using Http/2 with Hypercorn
### 3. Http/2 with Hypercorn
Use this if you want to run the proxy with hypercorn to support http/2
@ -731,7 +776,7 @@ docker run \
--run_hypercorn
```
### 4. Providing LiteLLM config.yaml file as a s3, GCS Bucket Object/url
### 4. config.yaml file on s3, GCS Bucket Object/url
Use this if you cannot mount a config file on your deployment service (example - AWS Fargate, Railway etc)
@ -787,7 +832,7 @@ docker run --name litellm-proxy \
<Tabs>
<TabItem value="AWS EKS" label="AWS EKS - Kubernetes">
### Kubernetes - Deploy on EKS
### Kubernetes (AWS EKS)
Step1. Create an EKS Cluster with the following spec
@ -880,7 +925,7 @@ Once the container is running, you can access the application by going to `http:
</TabItem>
<TabItem value="google-cloud-run" label="Google Cloud Run">
### Deploy on Google Cloud Run
### Google Cloud Run
1. Fork this repo - [github.com/BerriAI/example_litellm_gcp_cloud_run](https://github.com/BerriAI/example_litellm_gcp_cloud_run)
@ -907,7 +952,9 @@ curl https://litellm-7yjrj3ha2q-uc.a.run.app/v1/chat/completions \
</TabItem>
<TabItem value="render" label="Render deploy">
### Deploy on Render https://render.com/
### Render
https://render.com/
<iframe width="840" height="500" src="https://www.loom.com/embed/805964b3c8384b41be180a61442389a3" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
@ -916,7 +963,9 @@ curl https://litellm-7yjrj3ha2q-uc.a.run.app/v1/chat/completions \
</TabItem>
<TabItem value="railway" label="Railway">
### Deploy on Railway https://railway.app
### Railway
https://railway.app
**Step 1: Click the button** to deploy to Railway
@ -930,7 +979,7 @@ curl https://litellm-7yjrj3ha2q-uc.a.run.app/v1/chat/completions \
## Extras
### Run with docker compose
### Docker compose
**Step 1**
@ -999,3 +1048,4 @@ export DATABASE_SCHEMA="schema-name" # skip to use the default "public" schema
```bash
litellm --config /path/to/config.yaml --iam_token_db_auth
```

54
docs/my-website/docs/proxy/docker_quick_start.md
View file

@ -252,7 +252,7 @@ curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Content-Type: application/json' \
-d '{
"rpm_limit": 1
}
}'
```
[**See full API Spec**](https://litellm-api.up.railway.app/#/key%20management/generate_key_fn_key_generate_post)
@ -382,6 +382,56 @@ litellm_settings:
ssl_verify: false # 👈 KEY CHANGE
```
### (DB) All connection attempts failed
If you see:
```
httpx.ConnectError: All connection attempts failed
ERROR: Application startup failed. Exiting.
3:21:43 - LiteLLM Proxy:ERROR: utils.py:2207 - Error getting LiteLLM_SpendLogs row count: All connection attempts failed
```
This might be a DB permission issue.
1. Validate db user permission issue
Try creating a new database.
```bash
STATEMENT: CREATE DATABASE "litellm"
```
If you get:
```
ERROR: permission denied to create
```
This indicates you have a permission issue.
2. Grant permissions to your DB user
It should look something like this:
```
psql -U postgres
```
```
CREATE DATABASE litellm;
```
On CloudSQL, this is:
```
GRANT ALL PRIVILEGES ON DATABASE litellm TO your_username;
```
**What is `litellm_settings`?**
LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing) for handling LLM API calls.
@ -398,3 +448,5 @@ LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing
[![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw)

2
docs/my-website/docs/proxy/enterprise.md
View file

@ -17,12 +17,14 @@ Features:
- ✅ [JWT-Auth](../docs/proxy/token_auth.md)
- ✅ [Control available public, private routes (Restrict certain endpoints on proxy)](#control-available-public-private-routes)
- ✅ [Control available public, private routes](#control-available-public-private-routes)
- ✅ [Secret Managers - AWS Key Manager, Google Secret Manager, Azure Key, Hashicorp Vault](../secret)
- ✅ [[BETA] AWS Key Manager v2 - Key Decryption](#beta-aws-key-manager---key-decryption)
- ✅ IP address‑based access control lists
- ✅ Track Request IP Address
- ✅ [Use LiteLLM keys/authentication on Pass Through Endpoints](pass_through#✨-enterprise---use-litellm-keysauthentication-on-pass-through-endpoints)
- ✅ [Set Max Request Size / File Size on Requests](#set-max-request--response-size-on-litellm-proxy)
- ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](#enforce-required-params-for-llm-requests)
- ✅ [Key Rotations](./virtual_keys.md#-key-rotations)
- **Customize Logging, Guardrails, Caching per project**
- ✅ [Team Based Logging](./team_logging.md) - Allow each team to use their own Langfuse Project / custom callbacks
- ✅ [Disable Logging for a Team](./team_logging.md#disable-logging-for-a-team) - Switch off all logging for a team/project (GDPR Compliance)

153
docs/my-website/docs/proxy/guardrails/aim_security.md Normal file
View file

@ -0,0 +1,153 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Aim Security
## Quick Start
### 1. Create a new Aim Guard
Go to [Aim Application](https://app.aim.security/inventory/custom-ai-apps) and create a new guard.
When prompted, select API option, and name your guard.
:::note
In case you want to host your guard on-premise, you can enable this option
by [installing Aim Outpost](https://app.aim.security/settings/on-prem-deployment) prior to creating the guard.
:::
### 2. Configure your Aim Guard policies
In the newly created guard's page, you can find a reference to the prompt policy center of this guard.
You can decide which detections will be enabled, and set the threshold for each detection.
### 3. Add Aim Guardrail on your LiteLLM config.yaml
Define your guardrails under the `guardrails` section
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: aim-protected-app
litellm_params:
guardrail: aim
mode: pre_call
api_key: os.environ/AIM_API_KEY
api_base: os.environ/AIM_API_BASE # Optional, use only when using a self-hosted Aim Outpost
```
Under the `api_key`, insert the API key you were issued. The key can be found in the guard's page.
You can also set `AIM_API_KEY` as an environment variable.
By default, the `api_base` is set to `https://api.aim.security`. If you are using a self-hosted Aim Outpost, you can set the `api_base` to your Outpost's URL.
### 4. Start LiteLLM Gateway
```shell
litellm --config config.yaml
```
### 5. Make your first request
:::note
The following example depends on enabling *PII* detection in your guard.
You can adjust the request content to match different guard's policies.
:::
<Tabs>
<TabItem label="Successfully blocked request" value = "blocked">
:::note
When using LiteLLM with virtual keys, an `Authorization` header with the virtual key is required.
:::
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{"role": "user", "content": "hi my email is ishaan@berri.ai"}
],
"guardrails": ["aim-protected-app"]
}'
```
If configured correctly, since `ishaan@berri.ai` would be detected by the Aim Guard as PII, you'll receive a response similar to the following with a `400 Bad Request` status code:
```json
{
"error": {
"message": "\"ishaan@berri.ai\" detected as email",
"type": "None",
"param": "None",
"code": "400"
}
}
```
</TabItem>
<TabItem label="Successfully permitted request" value = "allowed">
:::note
When using LiteLLM with virtual keys, an `Authorization` header with the virtual key is required.
:::
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{"role": "user", "content": "hi what is the weather"}
],
"guardrails": ["aim-protected-app"]
}'
```
The above request should not be blocked, and you should receive a regular LLM response (simplified for brevity):
```json
{
"model": "gpt-3.5-turbo-0125",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "I can’t provide live weather updates without the internet. Let me know if you’d like general weather trends for a location and season instead!",
"role": "assistant"
}
}
]
}
```
</TabItem>
</Tabs>
# Advanced
Aim Guard provides user-specific Guardrail policies, enabling you to apply tailored policies to individual users.
To utilize this feature, include the end-user's email in the request payload by setting the `x-aim-user-email` header of your request.
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "x-aim-user-email: ishaan@berri.ai" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{"role": "user", "content": "hi what is the weather"}
],
"guardrails": ["aim-protected-app"]
}'
```

50
docs/my-website/docs/proxy/guardrails/quick_start.md
View file

@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Quick Start
# Guardrails - Quick Start
Setup Prompt Injection Detection, PII Masking on LiteLLM Proxy (AI Gateway)
@ -121,6 +121,49 @@ curl -i http://localhost:4000/v1/chat/completions \
</Tabs>
## **Default On Guardrails**
Set `default_on: true` in your guardrail config to run the guardrail on every request. This is useful if you want to run a guardrail on every request without the user having to specify it.
**Note:** These will run even if user specifies a different guardrail or empty guardrails array.
```yaml
guardrails:
- guardrail_name: "aporia-pre-guard"
litellm_params:
guardrail: aporia
mode: "pre_call"
default_on: true
```
**Test Request**
In this request, the guardrail `aporia-pre-guard` will run on every request because `default_on: true` is set.
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{"role": "user", "content": "hi my email is ishaan@berri.ai"}
]
}'
```
**Expected response**
Your response headers will incude `x-litellm-applied-guardrails` with the guardrail applied
```
x-litellm-applied-guardrails: aporia-pre-guard
```
## **Using Guardrails Client Side**
### Test yourself **(OSS)**
@ -349,7 +392,7 @@ Monitor which guardrails were executed and whether they passed or failed. e.g. g
### ✨ Control Guardrails per Project (API Key)
### ✨ Control Guardrails per API Key
:::info
@ -357,7 +400,7 @@ Monitor which guardrails were executed and whether they passed or failed. e.g. g
:::
Use this to control what guardrails run per project. In this tutorial we only want the following guardrails to run for 1 project (API Key)
Use this to control what guardrails run per API Key. In this tutorial we only want the following guardrails to run for 1 API Key
- `guardrails`: ["aporia-pre-guard", "aporia-post-guard"]
**Step 1** Create Key with guardrail settings
@ -484,6 +527,7 @@ guardrails:
mode: string # Required: One of "pre_call", "post_call", "during_call", "logging_only"
api_key: string # Required: API key for the guardrail service
api_base: string # Optional: Base URL for the guardrail service
default_on: boolean # Optional: Default False. When set to True, will run on every request, does not need client to specify guardrail in request
guardrail_info: # Optional[Dict]: Additional information about the guardrail
```

49
docs/my-website/docs/proxy/health.md
View file

@ -182,6 +182,28 @@ model_list:
mode: realtime
```
### Wildcard Routes
For wildcard routes, you can specify a `health_check_model` in your config.yaml. This model will be used for health checks for that wildcard route.
In this example, when running a health check for `openai/*`, the health check will make a `/chat/completions` request to `openai/gpt-4o-mini`.
```yaml
model_list:
- model_name: openai/*
litellm_params:
model: openai/*
api_key: os.environ/OPENAI_API_KEY
model_info:
health_check_model: openai/gpt-4o-mini
- model_name: anthropic/*
litellm_params:
model: anthropic/*
api_key: os.environ/ANTHROPIC_API_KEY
model_info:
health_check_model: anthropic/claude-3-5-sonnet-20240620
```
## Background Health Checks
You can enable model health checks being run in the background, to prevent each model from being queried too frequently via `/health`.
@ -223,6 +245,22 @@ general_settings:
health_check_details: False
```
## Health Check Timeout
The health check timeout is set in `litellm/constants.py` and defaults to 60 seconds.
This can be overridden in the config.yaml by setting `health_check_timeout` in the model_info section.
```yaml
model_list:
- model_name: openai/gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
model_info:
health_check_timeout: 10 # 👈 OVERRIDE HEALTH CHECK TIMEOUT
```
## `/health/readiness`
Unprotected endpoint for checking if proxy is ready to accept requests
@ -276,6 +314,17 @@ Example Response:
"I'm alive!"
```
## `/health/services`
Use this admin-only endpoint to check if a connected service (datadog/slack/langfuse/etc.) is healthy.
```bash
curl -L -X GET 'http://0.0.0.0:4000/health/services?service=datadog' -H 'Authorization: Bearer sk-1234'
```
[**API Reference**](https://litellm-api.up.railway.app/#/health/health_services_endpoint_health_services_get)
## Advanced - Call specific models
To check health of specific models, here's how to call them:

116
docs/my-website/docs/proxy/jwt_auth_arch.md Normal file
View file

@ -0,0 +1,116 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Control Model Access with OIDC (Azure AD/Keycloak/etc.)
:::info
✨ JWT Auth is on LiteLLM Enterprise
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Get free 7-day trial key](https://www.litellm.ai/#trial)
:::
<Image img={require('../../img/control_model_access_jwt.png')} style={{ width: '100%', maxWidth: '4000px' }} />
## Example Token
<Tabs>
<TabItem value="Azure AD">
```bash
{
"sub": "1234567890",
"name": "John Doe",
"email": "john.doe@example.com",
"roles": ["basic_user"] # 👈 ROLE
}
```
</TabItem>
<TabItem value="Keycloak">
```bash
{
"sub": "1234567890",
"name": "John Doe",
"email": "john.doe@example.com",
"resource_access": {
"litellm-test-client-id": {
"roles": ["basic_user"] # 👈 ROLE
}
}
}
```
</TabItem>
</Tabs>
## Proxy Configuration
<Tabs>
<TabItem value="Azure AD">
```yaml
general_settings:
enable_jwt_auth: True
litellm_jwtauth:
user_roles_jwt_field: "roles" # the field in the JWT that contains the roles
user_allowed_roles: ["basic_user"] # roles that map to an 'internal_user' role on LiteLLM
enforce_rbac: true # if true, will check if the user has the correct role to access the model
role_permissions: # control what models are allowed for each role
- role: internal_user
models: ["anthropic-claude"]
model_list:
- model: anthropic-claude
litellm_params:
model: claude-3-5-haiku-20241022
- model: openai-gpt-4o
litellm_params:
model: gpt-4o
```
</TabItem>
<TabItem value="Keycloak">
```yaml
general_settings:
enable_jwt_auth: True
litellm_jwtauth:
user_roles_jwt_field: "resource_access.litellm-test-client-id.roles" # the field in the JWT that contains the roles
user_allowed_roles: ["basic_user"] # roles that map to an 'internal_user' role on LiteLLM
enforce_rbac: true # if true, will check if the user has the correct role to access the model
role_permissions: # control what models are allowed for each role
- role: internal_user
models: ["anthropic-claude"]
model_list:
- model: anthropic-claude
litellm_params:
model: claude-3-5-haiku-20241022
- model: openai-gpt-4o
litellm_params:
model: gpt-4o
```
</TabItem>
</Tabs>
## How it works
1. Specify JWT_PUBLIC_KEY_URL - This is the public keys endpoint of your OpenID provider. For Azure AD it's `https://login.microsoftonline.com/{tenant_id}/discovery/v2.0/keys`. For Keycloak it's `{keycloak_base_url}/realms/{your-realm}/protocol/openid-connect/certs`.
1. Map JWT roles to LiteLLM roles - Done via `user_roles_jwt_field` and `user_allowed_roles`
- Currently just `internal_user` is supported for role mapping.
2. Specify model access:
- `role_permissions`: control what models are allowed for each role.
- `role`: the LiteLLM role to control access for. Allowed roles = ["internal_user", "proxy_admin", "team"]
- `models`: list of models that the role is allowed to access.
- `model_list`: parent list of models on the proxy. [Learn more](./configs.md#llm-configs-model_list)
3. Model Checks: The proxy will run validation checks on the received JWT. [Code](https://github.com/BerriAI/litellm/blob/3a4f5b23b5025b87b6d969f2485cc9bc741f9ba6/litellm/proxy/auth/user_api_key_auth.py#L284)

393
docs/my-website/docs/proxy/logging.md
View file

@ -5,6 +5,8 @@ Log Proxy input, output, and exceptions using:
- Langfuse
- OpenTelemetry
- GCS, s3, Azure (Blob) Buckets
- Lunary
- MLflow
- Custom Callbacks
- Langsmith
- DataDog
@ -109,6 +111,83 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
Removes any field with `user_api_key_*` from metadata.
### Turn off all tracking/logging
For some use cases, you may want to turn off all tracking/logging. You can do this by passing `no-log=True` in the request body.
:::info
Disable this by setting `global_disable_no_log_param:true` in your config.yaml file.
```yaml
litellm_settings:
global_disable_no_log_param: True
```
:::
<Tabs>
<TabItem value="Curl" label="Curl Request">
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer <litellm-api-key>' \
-d '{
"model": "openai/gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "What'\''s in this image?"
}
]
}
],
"max_tokens": 300,
"no-log": true # 👈 Key Change
}'
```
</TabItem>
<TabItem value="OpenAI" label="OpenAI">
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(
model="gpt-3.5-turbo",
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
extra_body={
"no-log": True # 👈 Key Change
}
)
print(response)
```
</TabItem>
</Tabs>
**Expected Console Log**
```
LiteLLM.Info: "no-log request, skipping logging"
```
## What gets logged?
Found under `kwargs["standard_logging_object"]`. This is a standard payload, logged for every response.
@ -267,6 +346,108 @@ print(response)
</TabItem>
</Tabs>
### Custom Tags
Set `tags` as part of your request body
<Tabs>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="sk-1234",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="llama3",
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
user="palantir",
extra_body={
"metadata": {
"tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]
}
}
)
print(response)
```
</TabItem>
<TabItem value="Curl" label="Curl Request">
Pass `metadata` as part of the request body
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "llama3",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
"user": "palantir",
"metadata": {
"tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]
}
}'
```
</TabItem>
<TabItem value="langchain" label="Langchain">
```python
from langchain.chat_models import ChatOpenAI
from langchain.prompts.chat import (
ChatPromptTemplate,
HumanMessagePromptTemplate,
SystemMessagePromptTemplate,
)
from langchain.schema import HumanMessage, SystemMessage
import os
os.environ["OPENAI_API_KEY"] = "sk-1234"
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:4000",
model = "llama3",
user="palantir",
extra_body={
"metadata": {
"tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]
}
}
)
messages = [
SystemMessage(
content="You are a helpful assistant that im using to make a test request to."
),
HumanMessage(
content="test from litellm. tell me why it's amazing in 1 sentence"
),
]
response = chat(messages)
print(response)
```
</TabItem>
</Tabs>
### LiteLLM Tags - `cache_hit`, `cache_key`
@ -854,6 +1035,74 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
6. Save the JSON file and add the path to `GCS_PATH_SERVICE_ACCOUNT`
## Google Cloud Storage - PubSub Topic
Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.google.com/pubsub/docs/reference/rest)
:::info
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
:::
| Property | Details |
|----------|---------|
| Description | Log LiteLLM `SpendLogs Table` to Google Cloud Storage PubSub Topic |
When to use `gcs_pubsub`?
- If your LiteLLM Database has crossed 1M+ spend logs and you want to send `SpendLogs` to a PubSub Topic that can be consumed by GCS BigQuery
#### Usage
1. Add `gcs_pubsub` to LiteLLM Config.yaml
```yaml
model_list:
- litellm_params:
api_base: https://exampleopenaiendpoint-production.up.railway.app/
api_key: my-fake-key
model: openai/my-fake-model
model_name: fake-openai-endpoint
litellm_settings:
callbacks: ["gcs_pubsub"] # 👈 KEY CHANGE # 👈 KEY CHANGE
```
2. Set required env variables
```shell
GCS_PUBSUB_TOPIC_ID="litellmDB"
GCS_PUBSUB_PROJECT_ID="reliableKeys"
```
3. Start Proxy
```
litellm --config /path/to/config.yaml
```
4. Test it!
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data ' {
"model": "fake-openai-endpoint",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}
'
```
## s3 Buckets
We will use the `--config` to set
@ -914,6 +1163,28 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
Your logs should be available on the specified s3 Bucket
### Team Alias Prefix in Object Key
**This is a preview feature**
You can add the team alias to the object key by setting the `team_alias` in the `config.yaml` file. This will prefix the object key with the team alias.
```yaml
litellm_settings:
callbacks: ["s3"]
enable_preview_features: true
s3_callback_params:
s3_bucket_name: logs-bucket-litellm
s3_region_name: us-west-2
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
s3_path: my-test-path
s3_endpoint_url: https://s3.amazonaws.com
s3_use_team_prefix: true
```
On s3 bucket, you will see the object key as `my-test-path/my-team-alias/...`
## Azure Blob Storage
Log LLM Logs to [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction)
@ -1003,6 +1274,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
LiteLLM Supports logging to the following Datdog Integrations:
- `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
- `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
- `ddtrace-run` [Datadog Tracing](#datadog-tracing)
<Tabs>
<TabItem value="datadog" label="Datadog Logs">
@ -1075,6 +1347,21 @@ Expected output on Datadog
<Image img={require('../../img/dd_small1.png')} />
#### Datadog Tracing
Use `ddtrace-run` to enable [Datadog Tracing](https://ddtrace.readthedocs.io/en/stable/installation_quickstart.html) on litellm proxy
Pass `USE_DDTRACE=true` to the docker run command. When `USE_DDTRACE=true`, the proxy will run `ddtrace-run litellm` as the `ENTRYPOINT` instead of just `litellm`
```bash
docker run \
-v $(pwd)/litellm_config.yaml:/app/config.yaml \
-e USE_DDTRACE=true \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
--config /app/config.yaml --detailed_debug
```
### Set DD variables (`DD_SERVICE` etc)
LiteLLM supports customizing the following Datadog environment variables
@ -1090,6 +1377,109 @@ LiteLLM supports customizing the following Datadog environment variables
| `HOSTNAME` | Hostname tag for your logs | "" | ❌ No |
| `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No |
## Lunary
#### Step1: Install dependencies and set your environment variables
Install the dependencies
```shell
pip install litellm lunary
```
Get you Lunary public key from from https://app.lunary.ai/settings
```shell
export LUNARY_PUBLIC_KEY="<your-public-key>"
```
#### Step 2: Create a `config.yaml` and set `lunary` callbacks
```yaml
model_list:
- model_name: "*"
litellm_params:
model: "*"
litellm_settings:
success_callback: ["lunary"]
failure_callback: ["lunary"]
```
#### Step 3: Start the LiteLLM proxy
```shell
litellm --config config.yaml
```
#### Step 4: Make a request
```shell
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o",
"messages": [
{
"role": "system",
"content": "You are a helpful math tutor. Guide the user through the solution step by step."
},
{
"role": "user",
"content": "how can I solve 8x + 7 = -23"
}
]
}'
```
## MLflow
#### Step1: Install dependencies
Install the dependencies.
```shell
pip install litellm mlflow
```
#### Step 2: Create a `config.yaml` with `mlflow` callback
```yaml
model_list:
- model_name: "*"
litellm_params:
model: "*"
litellm_settings:
success_callback: ["mlflow"]
failure_callback: ["mlflow"]
```
#### Step 3: Start the LiteLLM proxy
```shell
litellm --config config.yaml
```
#### Step 4: Make a request
```shell
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o-mini",
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
]
}'
```
#### Step 5: Review traces
Run the following command to start MLflow UI and review recorded traces.
```shell
mlflow ui
```
## Custom Callback Class [Async]
Use this when you want to run custom callbacks in `python`
@ -1114,9 +1504,6 @@ class MyCustomHandler(CustomLogger):
def log_post_api_call(self, kwargs, response_obj, start_time, end_time):
print(f"Post-API Call")
def log_stream_event(self, kwargs, response_obj, start_time, end_time):
print(f"On Stream")
def log_success_event(self, kwargs, response_obj, start_time, end_time):
print("On Success")

3
docs/my-website/docs/proxy/model_access.md
View file

@ -344,3 +344,6 @@ curl -i http://localhost:4000/v1/chat/completions \
</TabItem>
</Tabs>
## [Role Based Access Control (RBAC)](./jwt_auth_arch)

106
docs/my-website/docs/proxy/pagerduty.md Normal file
View file

@ -0,0 +1,106 @@
import Image from '@theme/IdealImage';
# PagerDuty Alerting
:::info
✨ PagerDuty Alerting is on LiteLLM Enterprise
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Get free 7-day trial key](https://www.litellm.ai/#trial)
:::
Handles two types of alerts:
- High LLM API Failure Rate. Configure X fails in Y seconds to trigger an alert.
- High Number of Hanging LLM Requests. Configure X hangs in Y seconds to trigger an alert.
## Quick Start
1. Set `PAGERDUTY_API_KEY="d8bxxxxx"` in your environment variables.
```
PAGERDUTY_API_KEY="d8bxxxxx"
```
2. Set PagerDuty Alerting in your config file.
```yaml
model_list:
- model_name: "openai/*"
litellm_params:
model: "openai/*"
api_key: os.environ/OPENAI_API_KEY
general_settings:
alerting: ["pagerduty"]
alerting_args:
failure_threshold: 1 # Number of requests failing in a window
failure_threshold_window_seconds: 10 # Window in seconds
# Requests hanging threshold
hanging_threshold_seconds: 0.0000001 # Number of seconds of waiting for a response before a request is considered hanging
hanging_threshold_window_seconds: 10 # Window in seconds
```
3. Test it
Start LiteLLM Proxy
```shell
litellm --config config.yaml
```
### LLM API Failure Alert
Try sending a bad request to proxy
```shell
curl -i --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data ' {
"model": "gpt-4o",
"user": "hi",
"messages": [
{
"role": "user",
"bad_param": "i like coffee"
}
]
}
'
```
<Image img={require('../../img/pagerduty_fail.png')} />
### LLM Hanging Alert
Try sending a hanging request to proxy
Since our hanging threshold is 0.0000001 seconds, you should see an alert.
```shell
curl -i --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data ' {
"model": "gpt-4o",
"user": "hi",
"messages": [
{
"role": "user",
"content": "i like coffee"
}
]
}
'
```
<Image img={require('../../img/pagerduty_hanging.png')} />

2
docs/my-website/docs/proxy/prod.md
View file

@ -133,7 +133,7 @@ To ensure only one service manages database migrations, use our [Helm PreSync ho
```yaml
db:
useExisting: true # use existing Postgres DB
url: postgresql://ishaanjaffer0324:3rnwpOBau6hT@ep-withered-mud-a5dkdpke.us-east-2.aws.neon.tech/test-argo-cd?sslmode=require # url of existing Postgres DB
url: postgresql://ishaanjaffer0324:... # url of existing Postgres DB
```
2. **LiteLLM Pods**:

116
docs/my-website/docs/proxy/prometheus.md
View file

@ -57,16 +57,52 @@ http://localhost:4000/metrics
# <proxy_base_url>/metrics
```
## Virtual Keys, Teams, Internal Users Metrics
## Virtual Keys, Teams, Internal Users
Use this for for tracking per [user, key, team, etc.](virtual_keys)
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_spend_metric` | Total Spend, per `"user", "key", "model", "team", "end-user"` |
| `litellm_total_tokens` | input + output tokens per `"user", "key", "model", "team", "end-user"` |
| `litellm_input_tokens` | input tokens per `"user", "key", "model", "team", "end-user"` |
| `litellm_output_tokens` | output tokens per `"user", "key", "model", "team", "end-user"` |
| `litellm_total_tokens` | input + output tokens per `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model"` |
| `litellm_input_tokens` | input tokens per `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model"` |
| `litellm_output_tokens` | output tokens per `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model"` |
### Team - Budget
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_team_max_budget_metric` | Max Budget for Team Labels: `"team_id", "team_alias"`|
| `litellm_remaining_team_budget_metric` | Remaining Budget for Team (A team created on LiteLLM) Labels: `"team_id", "team_alias"`|
| `litellm_team_budget_remaining_hours_metric` | Hours before the team budget is reset Labels: `"team_id", "team_alias"`|
### Virtual Key - Budget
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_api_key_max_budget_metric` | Max Budget for API Key Labels: `"hashed_api_key", "api_key_alias"`|
| `litellm_remaining_api_key_budget_metric` | Remaining Budget for API Key (A key Created on LiteLLM) Labels: `"hashed_api_key", "api_key_alias"`|
| `litellm_api_key_budget_remaining_hours_metric` | Hours before the API Key budget is reset Labels: `"hashed_api_key", "api_key_alias"`|
### Virtual Key - Rate Limit
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_remaining_api_key_requests_for_model` | Remaining Requests for a LiteLLM virtual API key, only if a model-specific rate limit (rpm) has been set for that virtual key. Labels: `"hashed_api_key", "api_key_alias", "model"`|
| `litellm_remaining_api_key_tokens_for_model` | Remaining Tokens for a LiteLLM virtual API key, only if a model-specific token limit (tpm) has been set for that virtual key. Labels: `"hashed_api_key", "api_key_alias", "model"`|
### Initialize Budget Metrics on Startup
If you want to initialize the key/team budget metrics on startup, you can set the `prometheus_initialize_budget_metrics` to `true` in the `config.yaml`
```yaml
litellm_settings:
callbacks: ["prometheus"]
prometheus_initialize_budget_metrics: true
```
## Proxy Level Tracking Metrics
@ -79,12 +115,11 @@ Use this to track overall LiteLLM Proxy usage.
| `litellm_proxy_failed_requests_metric` | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "exception_status", "exception_class"` |
| `litellm_proxy_total_requests_metric` | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code"` |
## LLM API / Provider Metrics
## LLM Provider Metrics
Use this for LLM API Error monitoring and tracking remaining rate limits and token limits
### Labels Tracked for LLM API Metrics
### Labels Tracked
| Label | Description |
|-------|-------------|
@ -100,7 +135,7 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
| exception_status | The status of the exception, if any |
| exception_class | The class of the exception, if any |
### Success and Failure Metrics for LLM API
### Success and Failure
| Metric Name | Description |
|----------------------|--------------------------------------|
@ -108,15 +143,14 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
| `litellm_deployment_failure_responses` | Total number of failed LLM API calls for a specific LLM deployment. Labels: `"requested_model", "litellm_model_name", "model_id", "api_base", "api_provider", "hashed_api_key", "api_key_alias", "team", "team_alias", "exception_status", "exception_class"` |
| `litellm_deployment_total_requests` | Total number of LLM API calls for deployment - success + failure. Labels: `"requested_model", "litellm_model_name", "model_id", "api_base", "api_provider", "hashed_api_key", "api_key_alias", "team", "team_alias"` |
### Remaining Requests and Tokens Metrics
### Remaining Requests and Tokens
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_remaining_requests_metric` | Track `x-ratelimit-remaining-requests` returned from LLM API Deployment. Labels: `"model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias"` |
| `litellm_remaining_tokens` | Track `x-ratelimit-remaining-tokens` return from LLM API Deployment. Labels: `"model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias"` |
### Deployment State Metrics
### Deployment State
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_deployment_state` | The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage. Labels: `"litellm_model_name", "model_id", "api_base", "api_provider"` |
@ -134,22 +168,60 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias` |
| `litellm_llm_api_latency_metric` | Latency (seconds) for just the LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias` |
| `litellm_request_total_latency_metric` | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model" |
| `litellm_overhead_latency_metric` | Latency overhead (seconds) added by LiteLLM processing - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model" |
| `litellm_llm_api_latency_metric` | Latency (seconds) for just the LLM API call - tracked for labels "model", "hashed_api_key", "api_key_alias", "team", "team_alias", "requested_model", "end_user", "user" |
| `litellm_llm_api_time_to_first_token_metric` | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias` [Note: only emitted for streaming requests] |
## Virtual Key - Budget, Rate Limit Metrics
## [BETA] Custom Metrics
Metrics used to track LiteLLM Proxy Budgeting and Rate limiting logic
Track custom metrics on prometheus on all events mentioned above.
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_remaining_team_budget_metric` | Remaining Budget for Team (A team created on LiteLLM) Labels: `"team_id", "team_alias"`|
| `litellm_remaining_api_key_budget_metric` | Remaining Budget for API Key (A key Created on LiteLLM) Labels: `"hashed_api_key", "api_key_alias"`|
| `litellm_remaining_api_key_requests_for_model` | Remaining Requests for a LiteLLM virtual API key, only if a model-specific rate limit (rpm) has been set for that virtual key. Labels: `"hashed_api_key", "api_key_alias", "model"`|
| `litellm_remaining_api_key_tokens_for_model` | Remaining Tokens for a LiteLLM virtual API key, only if a model-specific token limit (tpm) has been set for that virtual key. Labels: `"hashed_api_key", "api_key_alias", "model"`|
1. Define the custom metrics in the `config.yaml`
```yaml
model_list:
- model_name: openai/gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
callbacks: ["prometheus"]
custom_prometheus_metadata_labels: ["metadata.foo", "metadata.bar"]
```
2. Make a request with the custom metadata labels
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer <LITELLM_API_KEY>' \
-d '{
"model": "openai/gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "What's in this image?"
}
]
}
],
"max_tokens": 300,
"metadata": {
"foo": "hello world"
}
}'
```
3. Check your `/metrics` endpoint for the custom metrics
```
... "metadata_foo": "hello world" ...
```
## Monitor System Health
@ -170,6 +242,7 @@ litellm_settings:
| `litellm_redis_fails` | Number of failed redis calls |
| `litellm_self_latency` | Histogram latency for successful litellm api call |
## **🔥 LiteLLM Maintained Grafana Dashboards **
Link to Grafana Dashboards maintained by LiteLLM
@ -194,6 +267,7 @@ Here is a screenshot of the metrics you can monitor with the LiteLLM Grafana Das
| `litellm_requests_metric` | **deprecated** use `litellm_proxy_total_requests_metric` |
## FAQ
### What are `_created` vs. `_total` metrics?

20
docs/my-website/docs/proxy/prompt_management.md
View file

@ -2,12 +2,19 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Prompt Management
# [BETA] Prompt Management
:::info
This feature is currently in beta, and might change unexpectedly. We expect this to be more stable by next month (February 2025).
:::
Run experiments or change the specific model (e.g. from gpt-4o to gpt4o-mini finetune) from your prompt management tool (e.g. Langfuse) instead of making changes in the application.
Supported Integrations:
- [Langfuse](https://langfuse.com/docs/prompts/get-started)
- [Humanloop](../observability/humanloop)
## Quick Start
@ -42,11 +49,15 @@ resp = litellm.completion(
```yaml
model_list:
- model_name: gpt-3.5-turbo
- model_name: my-langfuse-model
litellm_params:
model: langfuse/gpt-3.5-turbo
model: langfuse/openai-model
prompt_id: "<langfuse_prompt_id>"
api_key: os.environ/OPENAI_API_KEY
- model_name: openai-model
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
```
2. Start the proxy
@ -65,7 +76,7 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gpt-3.5-turbo",
"model": "my-langfuse-model",
"messages": [
{
"role": "user",
@ -173,7 +184,6 @@ model_list:
- `prompt_variables`: A dictionary of variables that will be used to replace parts of the prompt.
## What is 'prompt_id'?
- `prompt_id`: The ID of the prompt that will be used for the request.

40
docs/my-website/docs/proxy/public_teams.md Normal file
View file

@ -0,0 +1,40 @@
# [BETA] Public Teams
Expose available teams to your users to join on signup.
<iframe width="840" height="500" src="https://www.loom.com/embed/7871ea15035a48d2a118b7486c2f7598?sid=267cd0ab-d92b-42fa-b97a-9f385ef8930c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## Quick Start
1. Create a team on LiteLLM
```bash
curl -X POST '<PROXY_BASE_URL>/team/new' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer <MASTER_KEY>' \
-d '{"name": "My Team", "team_id": "team_id_1"}'
```
2. Expose the team to your users
```yaml
litellm_settings:
default_internal_user_params:
available_teams: ["team_id_1"] # 👈 Make team available to new SSO users
```
3. Test it!
```bash
curl -L -X POST 'http://0.0.0.0:4000/team/member_add' \
-H 'Authorization: Bearer sk-<USER_KEY>' \
-H 'Content-Type: application/json' \
--data-raw '{
"team_id": "team_id_1",
"member": [{"role": "user", "user_id": "my-test-user"}]
}'
```

34
docs/my-website/docs/proxy/reliability.md
View file

@ -1007,7 +1007,34 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
}'
```
### Disable Fallbacks per key
### Disable Fallbacks (Per Request/Key)
<Tabs>
<TabItem value="request" label="Per Request">
You can disable fallbacks per key by setting `disable_fallbacks: true` in your request body.
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"messages": [
{
"role": "user",
"content": "List 5 important events in the XIX century"
}
],
"model": "gpt-3.5-turbo",
"disable_fallbacks": true # 👈 DISABLE FALLBACKS
}'
```
</TabItem>
<TabItem value="key" label="Per Key">
You can disable fallbacks per key by setting `disable_fallbacks: true` in your key metadata.
@ -1020,4 +1047,7 @@ curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
"disable_fallbacks": true
}
}'
```
```
</TabItem>
</Tabs>

12
docs/my-website/docs/proxy/request_headers.md Normal file
View file

@ -0,0 +1,12 @@
# Request Headers
Special headers that are supported by LiteLLM.
## LiteLLM Headers
`x-litellm-timeout` Optional[float]: The timeout for the request in seconds.
## Anthropic Headers
`anthropic-version` Optional[str]: The version of the Anthropic API to use.
`anthropic-beta` Optional[str]: The beta version of the Anthropic API to use.

43
docs/my-website/docs/proxy/self_serve.md
View file

@ -196,6 +196,49 @@ This budget does not apply to keys created under non-default teams.
[**Go Here**](./team_budgets.md)
### Auto-add SSO users to teams
1. Specify the JWT field that contains the team ids, that the user belongs to.
```yaml
general_settings:
master_key: sk-1234
litellm_jwtauth:
team_ids_jwt_field: "groups" # 👈 CAN BE ANY FIELD
```
This is assuming your SSO token looks like this:
```
{
...,
"groups": ["team_id_1", "team_id_2"]
}
```
2. Create the teams on LiteLLM
```bash
curl -X POST '<PROXY_BASE_URL>/team/new' \
-H 'Authorization: Bearer <PROXY_MASTER_KEY>' \
-H 'Content-Type: application/json' \
-D '{
"team_alias": "team_1",
"team_id": "team_id_1" # 👈 MUST BE THE SAME AS THE SSO GROUP ID
}'
```
3. Test the SSO flow
Here's a walkthrough of [how it works](https://www.loom.com/share/8959be458edf41fd85937452c29a33f3?sid=7ebd6d37-569a-4023-866e-e0cde67cb23e)
### Restrict Users from creating personal keys
This is useful if you only want users to create keys under a specific team.
This will also prevent users from using their session tokens on the test keys chat pane.
👉 [**See this**](./virtual_keys.md#restricting-key-generation)
## **All Settings for Self Serve / SSO Flow**
```yaml

74
docs/my-website/docs/proxy/temporary_budget_increase.md Normal file
View file

@ -0,0 +1,74 @@
# ✨ Temporary Budget Increase
Set temporary budget increase for a LiteLLM Virtual Key. Use this if you get asked to increase the budget for a key temporarily.
| Heirarchy | Supported |
|-----------|-----------|
| LiteLLM Virtual Key | ✅ |
| User | ❌ |
| Team | ❌ |
| Organization | ❌ |
:::note
✨ Temporary Budget Increase is a LiteLLM Enterprise feature.
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Get free 7-day trial key](https://www.litellm.ai/#trial)
:::
1. Create a LiteLLM Virtual Key with budget
```bash
curl -L -X POST 'http://localhost:4000/key/generate' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer LITELLM_MASTER_KEY' \
-d '{
"max_budget": 0.0000001
}'
```
Expected response:
```json
{
"key": "sk-your-new-key"
}
```
2. Update key with temporary budget increase
```bash
curl -L -X POST 'http://localhost:4000/key/update' \
-H 'Authorization: Bearer LITELLM_MASTER_KEY' \
-H 'Content-Type: application/json' \
-d '{
"key": "sk-your-new-key",
"temp_budget_increase": 100,
"temp_budget_expiry": "2025-01-15"
}'
```
3. Test it!
```bash
curl -L -X POST 'http://localhost:4000/chat/completions' \
-H 'Authorization: Bearer sk-your-new-key' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Hello, world!"}]
}'
```
Expected Response Header:
```
x-litellm-key-max-budget: 100.0000001
```

74
docs/my-website/docs/proxy/token_auth.md
View file

@ -1,7 +1,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# JWT-based Auth
# OIDC - JWT-based Auth
Use JWT's to auth admins / projects into the proxy.
@ -114,7 +114,7 @@ general_settings:
admin_jwt_scope: "litellm-proxy-admin"
```
## Advanced - Spend Tracking (End-Users / Internal Users / Team / Org)
## Tracking End-Users / Internal Users / Team / Org
Set the field in the jwt token, which corresponds to a litellm user / team / org.
@ -156,6 +156,76 @@ scope: ["litellm-proxy-admin",...]
scope: "litellm-proxy-admin ..."
```
## Control Model Access with Roles
Reject a JWT token if it's valid but doesn't have the required scopes / fields.
Only tokens which with valid Admin (`admin_jwt_scope`), User (`user_id_jwt_field`), Team (`team_id_jwt_field`) are allowed.
```yaml
general_settings:
enable_jwt_auth: True
litellm_jwtauth:
user_roles_jwt_field: "resource_access.litellm-test-client-id.roles"
user_allowed_roles: ["basic_user"] # roles that map to an 'internal_user' role on LiteLLM
enforce_rbac: true # if true, will check if the user has the correct role to access the model + endpoint
role_permissions: # control what models + endpointsare allowed for each role
- role: internal_user
models: ["anthropic-claude"]
```
**[Architecture Diagram (Control Model Access)](./jwt_auth_arch)**
## Control model access with Teams
1. Specify the JWT field that contains the team ids, that the user belongs to.
```yaml
general_settings:
master_key: sk-1234
litellm_jwtauth:
user_id_jwt_field: "sub"
team_ids_jwt_field: "groups"
```
This is assuming your token looks like this:
```
{
...,
"sub": "my-unique-user",
"groups": ["team_id_1", "team_id_2"]
}
```
2. Create the teams on LiteLLM
```bash
curl -X POST '<PROXY_BASE_URL>/team/new' \
-H 'Authorization: Bearer <PROXY_MASTER_KEY>' \
-H 'Content-Type: application/json' \
-D '{
"team_alias": "team_1",
"team_id": "team_id_1" # 👈 MUST BE THE SAME AS THE SSO GROUP ID
}'
```
3. Test the flow
SSO for UI: [**See Walkthrough**](https://www.loom.com/share/8959be458edf41fd85937452c29a33f3?sid=7ebd6d37-569a-4023-866e-e0cde67cb23e)
OIDC Auth for API: [**See Walkthrough**](https://www.loom.com/share/00fe2deab59a426183a46b1e2b522200?sid=4ed6d497-ead6-47f9-80c0-ca1c4b6b4814)
### Flow
- Validate if user id is in the DB (LiteLLM_UserTable)
- Validate if any of the groups are in the DB (LiteLLM_TeamTable)
- Validate if any group has model access
- If all checks pass, allow the request
## Advanced - Allowed Routes
Configure which routes a JWT can access via the config.

5
docs/my-website/docs/proxy/ui.md
View file

@ -6,11 +6,6 @@ import TabItem from '@theme/TabItem';
Create keys, track spend, add models without worrying about the config / CRUD endpoints.
:::info
This is in beta, so things may change. If you have feedback, [let us know](https://discord.com/invite/wuPM9dRgDw)
:::
<Image img={require('../../img/litellm_ui_create_key.png')} />

45
docs/my-website/docs/proxy/user_keys.md
View file

@ -381,6 +381,51 @@ assert user.age == 25
```
### **Streaming**
<Tabs>
<TabItem value="curl" label="curl">
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $OPTIONAL_YOUR_PROXY_KEY" \
-d '{
"model": "gpt-4-turbo",
"messages": [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
"stream": true
}'
```
</TabItem>
<TabItem value="sdk" label="SDK">
```python
from openai import OpenAI
client = OpenAI(
api_key="sk-1234", # [OPTIONAL] set if you set one on proxy, else set ""
base_url="http://0.0.0.0:4000",
)
messages = [{"role": "user", "content": "this is a test request, write a short poem"}]
completion = client.chat.completions.create(
model="gpt-4o",
messages=messages,
stream=True
)
print(completion)
```
</TabItem>
</Tabs>
### Function Calling
Here's some examples of doing function calling with the proxy.

104
docs/my-website/docs/proxy/virtual_keys.md
View file

@ -393,55 +393,6 @@ curl -L -X POST 'http://0.0.0.0:4000/key/unblock' \
```
### Custom Auth
You can now override the default api key auth.
Here's how:
#### 1. Create a custom auth file.
Make sure the response type follows the `UserAPIKeyAuth` pydantic object. This is used by for logging usage specific to that user key.
```python
from litellm.proxy._types import UserAPIKeyAuth
async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
try:
modified_master_key = "sk-my-master-key"
if api_key == modified_master_key:
return UserAPIKeyAuth(api_key=api_key)
raise Exception
except:
raise Exception
```
#### 2. Pass the filepath (relative to the config.yaml)
Pass the filepath to the config.yaml
e.g. if they're both in the same dir - `./config.yaml` and `./custom_auth.py`, this is what it looks like:
```yaml
model_list:
- model_name: "openai-model"
litellm_params:
model: "gpt-3.5-turbo"
litellm_settings:
drop_params: True
set_verbose: True
general_settings:
custom_auth: custom_auth.user_api_key_auth
```
[**Implementation Code**](https://github.com/BerriAI/litellm/blob/caf2a6b279ddbe89ebd1d8f4499f65715d684851/litellm/proxy/utils.py#L122)
#### 3. Start the proxy
```shell
$ litellm --config /path/to/config.yaml
```
### Custom /key/generate
If you need to add custom logic before generating a Proxy API Key (Example Validating `team_id`)
@ -568,6 +519,61 @@ litellm_settings:
team_id: "core-infra"
```
### ✨ Key Rotations
:::info
This is an Enterprise feature.
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Get free 7-day trial key](https://www.litellm.ai/#trial)
:::
Rotate an existing API Key, while optionally updating its parameters.
```bash
curl 'http://localhost:4000/key/sk-1234/regenerate' \
-X POST \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"max_budget": 100,
"metadata": {
"team": "core-infra"
},
"models": [
"gpt-4",
"gpt-3.5-turbo"
]
}'
```
**Read More**
- [Write rotated keys to secrets manager](https://docs.litellm.ai/docs/secret#aws-secret-manager)
[**👉 API REFERENCE DOCS**](https://litellm-api.up.railway.app/#/key%20management/regenerate_key_fn_key__key__regenerate_post)
### Temporary Budget Increase
Use the `/key/update` endpoint to increase the budget of an existing key.
```bash
curl -L -X POST 'http://localhost:4000/key/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{"key": "sk-b3Z3Lqdb_detHXSUp4ol4Q", "temp_budget_increase": 100, "temp_budget_expiry": "10d"}'
```
[API Reference](https://litellm-api.up.railway.app/#/key%20management/update_key_fn_key_update_post)
### Restricting Key Generation
Use this to control who can generate keys. Useful when letting others create keys on the UI.

5
docs/my-website/docs/scheduler.md
View file

@ -19,6 +19,11 @@ Prioritize LLM API requests in high-traffic.
- Priority - The lower the number, the higher the priority:
* e.g. `priority=0` > `priority=2000`
Supported Router endpoints:
- `acompletion` (`/v1/chat/completions` on Proxy)
- `atext_completion` (`/v1/completions` on Proxy)
## Quick Start
```python

215
docs/my-website/docs/secret.md
View file

@ -1,8 +1,8 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# Secret Manager
LiteLLM supports reading secrets from Azure Key Vault, Google Secret Manager
:::info
@ -14,6 +14,8 @@ LiteLLM supports reading secrets from Azure Key Vault, Google Secret Manager
:::
LiteLLM supports **reading secrets (eg. `OPENAI_API_KEY`)** and **writing secrets (eg. Virtual Keys)** from Azure Key Vault, Google Secret Manager, Hashicorp Vault, and AWS Secret Manager.
## Supported Secret Managers
- AWS Key Management Service
@ -21,38 +23,19 @@ LiteLLM supports reading secrets from Azure Key Vault, Google Secret Manager
- [Azure Key Vault](#azure-key-vault)
- [Google Secret Manager](#google-secret-manager)
- Google Key Management Service
- [Infisical Secret Manager](#infisical-secret-manager)
- [.env Files](#env-files)
## AWS Key Management V1
:::tip
[BETA] AWS Key Management v2 is on the enterprise tier. Go [here for docs](./proxy/enterprise.md#beta-aws-key-manager---key-decryption)
:::
Use AWS KMS to storing a hashed copy of your Proxy Master Key in the environment.
```bash
export LITELLM_MASTER_KEY="djZ9xjVaZ..." # 👈 ENCRYPTED KEY
export AWS_REGION_NAME="us-west-2"
```
```yaml
general_settings:
key_management_system: "aws_kms"
key_management_settings:
hosted_keys: ["LITELLM_MASTER_KEY"] # 👈 WHICH KEYS ARE STORED ON KMS
```
[**See Decryption Code**](https://github.com/BerriAI/litellm/blob/a2da2a8f168d45648b61279d4795d647d94f90c9/litellm/utils.py#L10182)
- [Hashicorp Vault](#hashicorp-vault)
## AWS Secret Manager
Store your proxy keys in AWS Secret Manager.
### Proxy Usage
| Feature | Support | Description |
|---------|----------|-------------|
| Reading Secrets | ✅ | Read secrets e.g `OPENAI_API_KEY` |
| Writing Secrets | ✅ | Store secrets e.g `Virtual Keys` |
#### Proxy Usage
1. Save AWS Credentials in your environment
```bash
@ -89,6 +72,20 @@ general_settings:
prefix_for_stored_virtual_keys: "litellm/" # OPTIONAL. If set, this prefix will be used for stored virtual keys in the secret manager
access_mode: "write_only" # Literal["read_only", "write_only", "read_and_write"]
```
</TabItem>
<TabItem value="read_and_write" label="Read + Write Keys with AWS Secret Manager">
```yaml
general_settings:
master_key: os.environ/litellm_master_key
key_management_system: "aws_secret_manager" # 👈 KEY CHANGE
key_management_settings:
store_virtual_keys: true # OPTIONAL. Defaults to False, when True will store virtual keys in secret manager
prefix_for_stored_virtual_keys: "litellm/" # OPTIONAL. If set, this prefix will be used for stored virtual keys in the secret manager
access_mode: "read_and_write" # Literal["read_only", "write_only", "read_and_write"]
hosted_keys: ["litellm_master_key"] # OPTIONAL. Specify which env keys you stored on AWS
```
</TabItem>
</Tabs>
@ -98,37 +95,113 @@ general_settings:
litellm --config /path/to/config.yaml
```
## Hashicorp Vault
| Feature | Support | Description |
|---------|----------|-------------|
| Reading Secrets | ✅ | Read secrets e.g `OPENAI_API_KEY` |
| Writing Secrets | ✅ | Store secrets e.g `Virtual Keys` |
Read secrets from [Hashicorp Vault](https://developer.hashicorp.com/vault/docs/secrets/kv/kv-v2)
**Step 1.** Add Hashicorp Vault details in your environment
LiteLLM supports two methods of authentication:
1. TLS cert authentication - `HCP_VAULT_CLIENT_CERT` and `HCP_VAULT_CLIENT_KEY`
2. Token authentication - `HCP_VAULT_TOKEN`
```bash
HCP_VAULT_ADDR="https://test-cluster-public-vault-0f98180c.e98296b2.z1.hashicorp.cloud:8200"
HCP_VAULT_NAMESPACE="admin"
# Authentication via TLS cert
HCP_VAULT_CLIENT_CERT="path/to/client.pem"
HCP_VAULT_CLIENT_KEY="path/to/client.key"
# OR - Authentication via token
HCP_VAULT_TOKEN="hvs.CAESIG52gL6ljBSdmq*****"
# OPTIONAL
HCP_VAULT_REFRESH_INTERVAL="86400" # defaults to 86400, frequency of cache refresh for Hashicorp Vault
```
**Step 2.** Add to proxy config.yaml
```yaml
general_settings:
key_management_system: "hashicorp_vault"
# [OPTIONAL SETTINGS]
key_management_settings:
store_virtual_keys: true # OPTIONAL. Defaults to False, when True will store virtual keys in secret manager
prefix_for_stored_virtual_keys: "litellm/" # OPTIONAL. If set, this prefix will be used for stored virtual keys in the secret manager
access_mode: "read_and_write" # Literal["read_only", "write_only", "read_and_write"]
```
**Step 3.** Start + test proxy
```
$ litellm --config /path/to/config.yaml
```
[Quick Test Proxy](./proxy/user_keys)
#### How it works
**Reading Secrets**
LiteLLM reads secrets from Hashicorp Vault's KV v2 engine using the following URL format:
```
{VAULT_ADDR}/v1/{NAMESPACE}/secret/data/{SECRET_NAME}
```
For example, if you have:
- `HCP_VAULT_ADDR="https://vault.example.com:8200"`
- `HCP_VAULT_NAMESPACE="admin"`
- Secret name: `AZURE_API_KEY`
LiteLLM will look up:
```
https://vault.example.com:8200/v1/admin/secret/data/AZURE_API_KEY
```
#### Expected Secret Format
LiteLLM expects all secrets to be stored as a JSON object with a `key` field containing the secret value.
For example, for `AZURE_API_KEY`, the secret should be stored as:
```json
{
"key": "sk-1234"
}
```
<Image img={require('../img/hcorp.png')} />
**Writing Secrets**
When a Virtual Key is Created / Deleted on LiteLLM, LiteLLM will automatically create / delete the secret in Hashicorp Vault.
- Create Virtual Key on LiteLLM either through the LiteLLM Admin UI or API
<Image img={require('../img/hcorp_create_virtual_key.png')} />
- Check Hashicorp Vault for secret
LiteLLM stores secret under the `prefix_for_stored_virtual_keys` path (default: `litellm/`)
<Image img={require('../img/hcorp_virtual_key.png')} />
## Azure Key Vault
<!--
### Quick Start
```python
### Instantiate Azure Key Vault Client ###
from azure.keyvault.secrets import SecretClient
from azure.identity import ClientSecretCredential
# Set your Azure Key Vault URI
KVUri = os.getenv("AZURE_KEY_VAULT_URI")
# Set your Azure AD application/client ID, client secret, and tenant ID - create an application with permission to call your key vault
client_id = os.getenv("AZURE_CLIENT_ID")
client_secret = os.getenv("AZURE_CLIENT_SECRET")
tenant_id = os.getenv("AZURE_TENANT_ID")
# Initialize the ClientSecretCredential
credential = ClientSecretCredential(client_id=client_id, client_secret=client_secret, tenant_id=tenant_id)
# Create the SecretClient using the credential
client = SecretClient(vault_url=KVUri, credential=credential)
### Connect to LiteLLM ###
import litellm
litellm.secret_manager = client
litellm.get_secret("your-test-key")
``` -->
### Usage with LiteLLM Proxy Server
#### Usage with LiteLLM Proxy Server
1. Install Proxy dependencies
```bash
@ -233,14 +306,36 @@ And in another terminal
$ litellm --test
```
[Quick Test Proxy](./proxy/quick_start#using-litellm-proxy---curl-request-openai-package-langchain-langchain-js)
[Quick Test Proxy](./proxy/user_keys)
<!--
## .env Files
If no secret manager client is specified, Litellm automatically uses the `.env` file to manage sensitive data. -->
## AWS Key Management V1
## All Secret Manager Settings
:::tip
[BETA] AWS Key Management v2 is on the enterprise tier. Go [here for docs](./proxy/enterprise.md#beta-aws-key-manager---key-decryption)
:::
Use AWS KMS to storing a hashed copy of your Proxy Master Key in the environment.
```bash
export LITELLM_MASTER_KEY="djZ9xjVaZ..." # 👈 ENCRYPTED KEY
export AWS_REGION_NAME="us-west-2"
```
```yaml
general_settings:
key_management_system: "aws_kms"
key_management_settings:
hosted_keys: ["LITELLM_MASTER_KEY"] # 👈 WHICH KEYS ARE STORED ON KMS
```
[**See Decryption Code**](https://github.com/BerriAI/litellm/blob/a2da2a8f168d45648b61279d4795d647d94f90c9/litellm/utils.py#L10182)
## **All Secret Manager Settings**
All settings related to secret management

16
docs/my-website/docs/set_keys.md
View file

@ -179,6 +179,22 @@ assert(valid_models == expected_models)
os.environ = old_environ
```
### `get_valid_models(check_provider_endpoint: True)`
This helper will check the provider's endpoint for valid models.
Currently implemented for:
- OpenAI (if OPENAI_API_KEY is set)
- Fireworks AI (if FIREWORKS_AI_API_KEY is set)
- LiteLLM Proxy (if LITELLM_PROXY_API_KEY is set)
```python
from litellm import get_valid_models
valid_models = get_valid_models(check_provider_endpoint=True)
print(valid_models)
```
### `validate_environment(model: str)`
This helper tells you if you have all the required environment variables for a model, and if not - what's missing.

8
docs/my-website/docs/tutorials/eval_suites.md
View file

@ -2,9 +2,9 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Evaluate LLMs - ML Flow Evals, Auto Eval
# Evaluate LLMs - MLflow Evals, Auto Eval
## Using LiteLLM with ML Flow
## Using LiteLLM with MLflow
MLflow provides an API `mlflow.evaluate()` to help evaluate your LLMs https://mlflow.org/docs/latest/llms/llm-evaluate/index.html
### Pre Requisites
@ -153,7 +153,7 @@ $ litellm --model command-nightly
</Tabs>
### Step 2: Run ML Flow
### Step 2: Run MLflow
Before running the eval we will set `openai.api_base` to the litellm proxy from Step 1
```python
@ -209,7 +209,7 @@ with mlflow.start_run() as run:
```
### ML Flow Output
### MLflow Output
```
{'toxicity/v1/mean': 0.00014476531214313582, 'toxicity/v1/variance': 2.5759661361262862e-12, 'toxicity/v1/p90': 0.00014604929747292773, 'toxicity/v1/ratio': 0.0, 'exact_match/v1': 0.0}
Downloading artifacts: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 1890.18it/s]

49
docs/my-website/docs/tutorials/instructor.md
View file

@ -1,32 +1,22 @@
# Instructor - Function Calling
Use LiteLLM Router with [jxnl's instructor library](https://github.com/jxnl/instructor) for function calling in prod.
Use LiteLLM with [jxnl's instructor library](https://github.com/jxnl/instructor) for function calling in prod.
## Usage
```python
import litellm
from litellm import Router
import os
import instructor
from litellm import completion
from pydantic import BaseModel
litellm.set_verbose = True # 👈 print DEBUG LOGS
os.environ["LITELLM_LOG"] = "DEBUG" # 👈 print DEBUG LOGS
client = instructor.patch(
Router(
model_list=[
{
"model_name": "gpt-3.5-turbo", openai model name
"litellm_params": { # params for litellm completion/embedding call - e.g.: https://github.com/BerriAI/litellm/blob/62a591f90c99120e1a51a8445f5c3752586868ea/litellm/router.py#L111
"model": "azure/chatgpt-v-2",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
},
}
]
)
)
client = instructor.from_litellm(completion)
# import dotenv
# dotenv.load_dotenv()
class UserDetail(BaseModel):
@ -35,7 +25,7 @@ class UserDetail(BaseModel):
user = client.chat.completions.create(
model="gpt-3.5-turbo",
model="gpt-4o-mini",
response_model=UserDetail,
messages=[
{"role": "user", "content": "Extract Jason is 25 years old"},
@ -52,25 +42,20 @@ print(f"user: {user}")
## Async Calls
```python
import litellm
import asyncio
import instructor
from litellm import Router
import instructor, asyncio
from pydantic import BaseModel
aclient = instructor.apatch(
aclient = instructor.patch(
Router(
model_list=[
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/chatgpt-v-2",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
},
"model_name": "gpt-4o-mini",
"litellm_params": {"model": "gpt-4o-mini"},
}
],
default_litellm_params={"acompletion": True}, # 👈 IMPORTANT - tells litellm to route to async completion function.
default_litellm_params={"acompletion": True}, # 👈 IMPORTANT - tells litellm to route to async completion function.
)
)
@ -82,7 +67,7 @@ class UserExtract(BaseModel):
async def main():
model = await aclient.chat.completions.create(
model="gpt-3.5-turbo",
model="gpt-4o-mini",
response_model=UserExtract,
messages=[
{"role": "user", "content": "Extract jason is 25 years old"},

21
docs/my-website/docusaurus.config.js
View file

@ -40,17 +40,16 @@ const config = {
[
'@docusaurus/plugin-content-blog',
{
id: 'release_notes',
path: './release_notes', // Folder where your release notes are stored
routeBasePath: '/release_notes', // URL path for the release notes
sortPosts: (a, b) => {
// Extract folder names from the file paths
const folderA = a.metadata.permalink.split('/')[2]; // Get folder name from permalink
const folderB = b.metadata.permalink.split('/')[2];
return folderA.localeCompare(folderB); // Compare folder names
},
include: ['**/*.md', '**/*.mdx'], // Files to include
// Other blog options
id: 'release_notes',
path: './release_notes',
routeBasePath: 'release_notes',
blogTitle: 'Release Notes',
blogSidebarTitle: 'All Releases',
blogSidebarCount: 'ALL',
postsPerPage: 'ALL',
showReadingTime: false,
sortPosts: 'descending',
include: ['**/*.{md,mdx}'],
},
],

BIN
docs/my-website/img/10_instance_proxy.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 158 KiB

BIN
docs/my-website/img/1_instance_proxy.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 156 KiB

BIN
docs/my-website/img/2_instance_proxy.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 158 KiB

BIN
docs/my-website/img/control_model_access_jwt.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 113 KiB

BIN
docs/my-website/img/hcorp.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 158 KiB

BIN
docs/my-website/img/hcorp_create_virtual_key.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 190 KiB

BIN
docs/my-website/img/hcorp_virtual_key.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 145 KiB

BIN
docs/my-website/img/instances_vs_rps.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 150 KiB

Some files were not shown because too many files have changed in this diff Show more